Merge branch 'develop' into docker-python-requirements

.dredd/hooks/capabilities.go

@@ -2,11 +2,12 @@ package main
 
 import (
 	"encoding/json"
-	"github.com/ansible-semaphore/semaphore/db"
-	trans "github.com/snikch/goodman/transaction"
 	"regexp"
 	"strconv"
 	"strings"
+
+	"github.com/ansible-semaphore/semaphore/db"
+	trans "github.com/snikch/goodman/transaction"
 )
 
 // STATE
@@ -18,23 +19,32 @@ var userKey *db.AccessKey
 var task *db.Task
 var schedule *db.Schedule
 var view *db.View
+var integration *db.Integration
+var integrationextractvalue *db.IntegrationExtractValue
+var integrationmatch *db.IntegrationMatcher
 
 // Runtime created simple ID values for some items we need to reference in other objects
 var repoID int
 var inventoryID int
 var environmentID int
 var templateID int
+var integrationID int
+var integrationExtractValueID int
+var integrationMatchID int
 
 var capabilities = map[string][]string{
-	"user":        {},
+	"user":                    {},
-	"project":     {"user"},
+	"project":                 {"user"},
-	"repository":  {"access_key"},
+	"repository":              {"access_key"},
-	"inventory":   {"repository"},
+	"inventory":               {"repository"},
-	"environment": {"repository"},
+	"environment":             {"repository"},
-	"template":    {"repository", "inventory", "environment", "view"},
+	"template":                {"repository", "inventory", "environment", "view"},
-	"task":        {"template"},
+	"task":                    {"template"},
-	"schedule":    {"template"},
+	"schedule":                {"template"},
-	"view":        {},
+	"view":                    {},
+	"integration":             {"project", "template"},
+	"integrationextractvalue": {"integration"},
+	"integrationmatcher":      {"integration"},
 }
 
 func capabilityWrapper(cap string) func(t *trans.Transaction) {
@@ -131,6 +141,15 @@ func resolveCapability(caps []string, resolved []string, uid string) {
 			templateID = res.ID
 		case "task":
 			task = addTask()
+		case "integration":
+			integration = addIntegration()
+			integrationID = integration.ID
+		case "integrationextractvalue":
+			integrationextractvalue = addIntegrationExtractValue()
+			integrationExtractValueID = integrationextractvalue.ID
+		case "integrationmatcher":
+			integrationmatch = addIntegrationMatcher()
+			integrationMatchID = integrationmatch.ID
 		default:
 			panic("unknown capability " + v)
 		}
@@ -150,13 +169,16 @@ var pathSubPatterns = []func() string{
 	func() string { return strconv.Itoa(userProject.ID) },
 	func() string { return strconv.Itoa(userPathTestUser.ID) },
 	func() string { return strconv.Itoa(userKey.ID) },
-	func() string { return strconv.Itoa(int(repoID)) },
+	func() string { return strconv.Itoa(repoID) },
-	func() string { return strconv.Itoa(int(inventoryID)) },
+	func() string { return strconv.Itoa(inventoryID) },
-	func() string { return strconv.Itoa(int(environmentID)) },
+	func() string { return strconv.Itoa(environmentID) },
-	func() string { return strconv.Itoa(int(templateID)) },
+	func() string { return strconv.Itoa(templateID) },
 	func() string { return strconv.Itoa(task.ID) },
 	func() string { return strconv.Itoa(schedule.ID) },
 	func() string { return strconv.Itoa(view.ID) },
+	func() string { return strconv.Itoa(integration.ID) },
+	func() string { return strconv.Itoa(integrationextractvalue.ID) },
+	func() string { return strconv.Itoa(integrationmatch.ID) },
 }
 
 // alterRequestPath with the above slice of functions
@@ -165,12 +187,14 @@ func alterRequestPath(t *trans.Transaction) {
 	exploded := make([]string, len(pathArgs))
 	copy(exploded, pathArgs)
 	for k, v := range pathSubPatterns {
+
 		pos, exists := stringInSlice(strconv.Itoa(k+1), exploded)
 		if exists {
 			pathArgs[pos] = v()
 		}
 	}
 	t.FullPath = strings.Join(pathArgs, "/")
+
 	t.Request.URI = t.FullPath
 }
 
@@ -198,9 +222,21 @@ func alterRequestBody(t *trans.Transaction) {
 	if view != nil {
 		bodyFieldProcessor("view_id", view.ID, &request)
 	}
+
+	if integration != nil {
+		bodyFieldProcessor("integration_id", integration.ID, &request)
+	}
+	if integrationextractvalue != nil {
+		bodyFieldProcessor("value_id", integrationextractvalue.ID, &request)
+	}
+	if integrationmatch != nil {
+		bodyFieldProcessor("matcher_id", integrationmatch.ID, &request)
+	}
+
 	// Inject object ID to body for PUT requests
 	if strings.ToLower(t.Request.Method) == "put" {
-		putRequestPathRE := regexp.MustCompile(`/api/(?:project/\d+/)?\w+/(\d+)/?$`)
+
+		putRequestPathRE := regexp.MustCompile(`\w+/(\d+)/?$`)
 		m := putRequestPathRE.FindStringSubmatch(t.FullPath)
 		if len(m) > 0 {
 			objectID, err := strconv.Atoi(m[1])

.dredd/hooks/helpers.go

@@ -3,6 +3,10 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"math/rand"
+	"os"
+	"time"
+
 	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/ansible-semaphore/semaphore/db/bolt"
 	"github.com/ansible-semaphore/semaphore/db/factory"
@@ -10,9 +14,6 @@ import (
 	"github.com/ansible-semaphore/semaphore/util"
 	"github.com/go-gorp/gorp/v3"
 	"github.com/snikch/goodman/transaction"
-	"math/rand"
-	"os"
-	"time"
 )
 
 // Test Runner User
@@ -59,6 +60,9 @@ func truncateAll() {
 		"project__user",
 		"user",
 		"project__view",
+		"project__integration",
+		"project__integration_extract_value",
+		"project__integration_matcher",
 	}
 
 	switch store.(type) {
@@ -107,7 +111,7 @@ func addUserProjectRelation(pid int, user int) {
 	_, err := store.CreateProjectUser(db.ProjectUser{
 		ProjectID: pid,
 		UserID:    user,
-		Admin:     true,
+		Role:      db.ProjectOwner,
 	})
 	if err != nil {
 		panic(err)
@@ -225,6 +229,54 @@ func addTask() *db.Task {
 	return &t
 }
 
+func addIntegration() *db.Integration {
+	integration, err := store.CreateIntegration(db.Integration{
+		ProjectID:  userProject.ID,
+		Name:       "Test Integration",
+		TemplateID: templateID,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return &integration
+}
+
+func addIntegrationExtractValue() *db.IntegrationExtractValue {
+	integrationextractvalue, err := store.CreateIntegrationExtractValue(userProject.ID, db.IntegrationExtractValue{
+		Name:          "Value",
+		IntegrationID: integrationID,
+		ValueSource:   db.IntegrationExtractBodyValue,
+		BodyDataType:  db.IntegrationBodyDataJSON,
+		Key:           "key",
+		Variable:      "var",
+	})
+
+	if err != nil {
+		panic(err)
+	}
+
+	return &integrationextractvalue
+}
+
+func addIntegrationMatcher() *db.IntegrationMatcher {
+	integrationmatch, err := store.CreateIntegrationMatcher(userProject.ID, db.IntegrationMatcher{
+		Name:          "matcher",
+		IntegrationID: integrationID,
+		MatchType:     "body",
+		Method:        "equals",
+		BodyDataType:  "json",
+		Key:           "key",
+		Value:         "value",
+	})
+
+	if err != nil {
+		panic(err)
+	}
+
+	return &integrationmatch
+}
+
 // Token Handling
 func addToken(tok string, user int) {
 	_, err := store.CreateAPIToken(db.APIToken{

.dredd/hooks/main.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"github.com/snikch/goodman/hooks"
-	trans "github.com/snikch/goodman/transaction"
 	"strconv"
 	"strings"
+
+	"github.com/snikch/goodman/hooks"
+	trans "github.com/snikch/goodman/transaction"
 )
 
 const (
@@ -57,6 +58,7 @@ func main() {
 		defer store.Close("")
 		addToken(expiredToken, testRunnerUser.ID)
 	})
+
 	h.After("user > /api/user/tokens/{api_token_id} > Expires API token > 204 > application/json", func(transaction *trans.Transaction) {
 		dbConnect()
 		defer store.Close("")
@@ -74,13 +76,28 @@ func main() {
 		dbConnect()
 		defer store.Close("")
 		deleteUserProjectRelation(userProject.ID, userPathTestUser.ID)
-		transaction.Request.Body = "{ \"user_id\": " + strconv.Itoa(userPathTestUser.ID) + ",\"admin\": true}"
+		transaction.Request.Body = "{ \"user_id\": " + strconv.Itoa(userPathTestUser.ID) + ",\"role\": \"owner\"}"
 	})
 
+	h.Before("project > /api/project/{project_id}/integrations > get all integrations > 200 > application/json", capabilityWrapper("integration"))
+	h.Before("project > /api/project/{project_id}/integrations/{integration_id} > Get Integration > 200 > application/json", capabilityWrapper("integration"))
+	h.Before("project > /api/project/{project_id}/integrations/{integration_id} > Update Integration > 204 > application/json", capabilityWrapper("integration"))
+	h.Before("project > /api/project/{project_id}/integrations/{integration_id} > Remove integration > 204 > application/json", capabilityWrapper("integration"))
+
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/values > Get Integration Extracted Values linked to integration extractor > 200 > application/json", capabilityWrapper("integrationextractvalue"))
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/values > Add Integration Extracted Value > 204 > application/json", capabilityWrapper("integrationextractvalue"))
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/values/{extractvalue_id} > Removes integration extract value > 204 > application/json", capabilityWrapper("integrationextractvalue"))
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/values > Add Integration Extracted Value > 204 > application/json", capabilityWrapper("integration"))
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/values/{extractvalue_id} > Updates Integration ExtractValue > 204 > application/json", capabilityWrapper("integrationextractvalue"))
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/matchers > Get Integration Matcher linked to integration extractor > 200 > application/json", capabilityWrapper("integration"))
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/matchers > Add Integration Matcher > 204 > application/json", capabilityWrapper("integration"))
+	h.Before("integration > /api/project/{project_id}/integrations/{integration_id}/matchers/{matcher_id} > Updates Integration Matcher > 204 > application/json", capabilityWrapper("integrationmatcher"))
+
 	h.Before("project > /api/project/{project_id}/keys/{key_id} > Updates access key > 204 > application/json", capabilityWrapper("access_key"))
 	h.Before("project > /api/project/{project_id}/keys/{key_id} > Removes access key > 204 > application/json", capabilityWrapper("access_key"))
 
 	h.Before("project > /api/project/{project_id}/repositories > Add repository > 204 > application/json", capabilityWrapper("access_key"))
+	h.Before("project > /api/project/{project_id}/repositories/{repository_id} > Updates repository > 204 > application/json", capabilityWrapper("repository"))
 	h.Before("project > /api/project/{project_id}/repositories/{repository_id} > Removes repository > 204 > application/json", capabilityWrapper("repository"))
 
 	h.Before("project > /api/project/{project_id}/inventory > create inventory > 201 > application/json", capabilityWrapper("inventory"))
@@ -104,6 +121,7 @@ func main() {
 	h.Before("project > /api/project/{project_id}/tasks/{task_id} > Get a single task > 200 > application/json", capabilityWrapper("task"))
 	h.Before("project > /api/project/{project_id}/tasks/{task_id} > Deletes task (including output) > 204 > application/json", capabilityWrapper("task"))
 	h.Before("project > /api/project/{project_id}/tasks/{task_id}/output > Get task output > 200 > application/json", capabilityWrapper("task"))
+	h.Before("project > /api/project/{project_id}/tasks/{task_id}/stop > Stop a job > 204 > application/json", capabilityWrapper("task"))
 
 	h.Before("schedule > /api/project/{project_id}/schedules/{schedule_id} > Get schedule > 200 > application/json", capabilityWrapper("schedule"))
 	h.Before("schedule > /api/project/{project_id}/schedules/{schedule_id} > Updates schedule > 204 > application/json", capabilityWrapper("schedule"))
@@ -113,6 +131,10 @@ func main() {
 	h.Before("project > /api/project/{project_id}/views/{view_id} > Updates view > 204 > application/json", capabilityWrapper("view"))
 	h.Before("project > /api/project/{project_id}/views/{view_id} > Removes view > 204 > application/json", capabilityWrapper("view"))
 
+	h.Before("project > /api/project/{project_id}/backup > Get backup > 200 > application/json", func(t *trans.Transaction) {
+		addCapabilities([]string{"repository", "inventory", "environment", "view", "template"})
+	})
+
 	//Add these last as they normalize the requests and path values after hook processing
 	h.BeforeAll(func(transactions []*trans.Transaction) {
 		for _, t := range transactions {

.github/FUNDING.yml

@@ -2,7 +2,7 @@
 
 github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
 patreon: # Replace with a single Patreon username
-open_collective: semaphore
+open_collective: # semaphore
 ko_fi: fiftin
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry

.github/ISSUE_TEMPLATES/documentation.yml

@@ -0,0 +1,48 @@
+---
+
+name: Documentation
+description: You have a found missing or invalid documentation
+title: "Docs: "
+labels: ['documentation', 'triage']
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please make sure to go through these steps **before opening an issue**:
+        
+        - [ ] Read the [documentation](https://docs.ansible-semaphore.com/)
+          - [ ] Read the [troubleshooting guide](https://docs.ansible-semaphore.com/administration-guide/troubleshooting)
+          - [ ] Read the [documentation regarding manual installations](https://docs.ansible-semaphore.com/administration-guide/installation_manually) if you did install Semaphore that way
+
+        - [ ] Check if there are existing [issues](https://github.com/ansible-semaphore/semaphore/issues) or [discussions](https://github.com/ansible-semaphore/semaphore/discussions) regarding your topic
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: |
+        Describe what part of the documentation is missing or wrong!
+        Please also tell us what you would expected to find.
+        What would you change or add?
+    validations:
+      required: true
+
+  - type: dropdown
+    id: related-to
+    attributes:
+      label: Related to
+      description: |
+        To what parts of Semaphore is the documentation related? (if any)
+
+      multiple: true
+      options:
+        - Web-Frontend (what users interact with)
+        - Web-Backend (APIs)
+        - Service (scheduled tasks, alerts)
+        - Ansible (task execution)
+        - Configuration
+        - Database
+        - Docker
+    validations:
+      required: false

.github/ISSUE_TEMPLATES/feature_request.yml

@@ -0,0 +1,94 @@
+---
+
+name: Feature request
+description: You would like to have a new feature implemented
+title: "Feature: "
+labels: ['feature', 'triage']
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please make sure to go through these steps **before opening an issue**:
+        
+        - [ ] Read the [documentation](https://docs.ansible-semaphore.com/)
+          - [ ] Read the [troubleshooting guide](https://docs.ansible-semaphore.com/administration-guide/troubleshooting)
+          - [ ] Read the [documentation regarding manual installations](https://docs.ansible-semaphore.com/administration-guide/installation_manually) if you did install Semaphore that way
+
+        - [ ] Check if there are existing [issues](https://github.com/ansible-semaphore/semaphore/issues) or [discussions](https://github.com/ansible-semaphore/semaphore/discussions) regarding your topic
+
+  - type: dropdown
+    id: related-to
+    attributes:
+      label: Related to
+      description: |
+        To what parts of Semaphore is the feature related?
+
+      multiple: true
+      options:
+        - Web-Frontend (what users interact with)
+        - Web-Backend (APIs)
+        - Service (scheduled tasks, alerts)
+        - Ansible (task execution)
+        - Configuration
+        - Database
+        - Docker
+        - Other
+    validations:
+      required: true
+
+  - type: dropdown
+    id: impact
+    attributes:
+      label: Impact
+      description: |
+        What impact would the feature have for Semaphore users?
+
+      multiple: false
+      options:
+        - nice to have
+        - nice to have for enterprise usage
+        - better user experience
+        - security improvements
+        - major improvement to user experience
+        - must have for enterprise usage
+        - must have
+    validations:
+      required: true
+
+  - type: textarea
+    id: feature
+    attributes:
+      label: Missing Feature
+      description: |
+        Describe the feature you are missing.
+        Why would you like to see such a feature being implemented?
+
+    validations:
+      required: true
+
+  - type: textarea
+    id: implementation
+    attributes:
+      label: Implementation
+      description: |
+        Please think about how the feature should be implemented.
+        What would you suggest?
+        How should it look and behave?
+
+    validations:
+      required: true
+
+  - type: textarea
+    id: design
+    attributes:
+      label: Design
+      description: |
+        If you have programming experience yourself:
+        Please provide us with an example how you would design this feature.
+
+        What edge-cases need to be covered?
+        Are there relations to other components that need to be though of?
+
+    validations:
+      required: false

.github/ISSUE_TEMPLATES/problem.yml

@@ -0,0 +1,169 @@
+---
+
+name: Problem
+description: You have encountered problems when using Semaphore
+title: "Problem: "
+labels: ['problem', 'triage']
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please make sure to go through these steps **before opening an issue**:
+        
+        - [ ] Read the [documentation](https://docs.ansible-semaphore.com/)
+          - [ ] Read the [troubleshooting guide](https://docs.ansible-semaphore.com/administration-guide/troubleshooting)
+          - [ ] Read the [documentation regarding manual installations](https://docs.ansible-semaphore.com/administration-guide/installation_manually) if you don't use docker
+
+        - [ ] Check if there are existing [issues](https://github.com/ansible-semaphore/semaphore/issues) or [discussions](https://github.com/ansible-semaphore/semaphore/discussions) regarding your topic
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Issue
+      description: |
+        Describe the problem you encountered and tell us what you would have expected to happen
+
+    validations:
+      required: true
+
+  - type: dropdown
+    id: impact
+    attributes:
+      label: Impact
+      description: |
+        What parts of Semaphore are impacted by the problem?
+
+      multiple: true
+      options:
+        - Web-Frontend (what users interact with)
+        - Web-Backend (APIs)
+        - Service (scheduled tasks, alerts)
+        - Ansible (task execution)
+        - Configuration
+        - Database
+        - Docker
+        - Semaphore Project
+        - Other
+    validations:
+      required: true
+
+  - type: dropdown
+    id: install-method
+    attributes:
+      label: Installation method
+      description: |
+        How did you install Semaphore?
+
+      multiple: false
+      options:
+        - Docker
+        - Package
+        - Binary
+        - Snap
+    validations:
+      required: true
+
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: Browser
+      description: |
+        If the problem occurs in the Semaphore WebUI - in what browsers do you see it?
+
+      multiple: true
+      options:
+        - Firefox
+        - Chrome
+        - Safari
+        - Microsoft Edge
+        - Opera
+
+  - type: textarea
+    id: version-semaphore
+    attributes:
+      label: Semaphore Version
+      description: |
+        What version of Semaphore are you running?
+        > Command: `semaphore version`
+    validations:
+      required: true
+
+  - type: textarea
+    id: version-ansible
+    attributes:
+      label: Ansible Version
+      description: |
+        If your problem occurs when executing a task:
+        > What version of Ansible are you running?
+        > Command: `ansible --version`
+
+        If your problem occurs when executing a specific Ansible Module:
+        > Provide the Ansible Module versions!
+        > Command: `ansible-galaxy collection list`
+
+      render: bash
+    validations:
+      required: false
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs & errors
+      description: |
+        Provide logs and error messages you have encountered!
+        
+        Logs of the service:
+        > Docker command: `docker logs <container-name>`
+        > Systemd command: `journalctl -u <serivce-name> --no-pager --full -n 250`
+
+        If the error occurs in the WebUI:
+        > please add a screenshot
+        > check your browser console for errors (`F12` in most browsers)
+
+        Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
+
+    validations:
+      required: false
+
+  - type: textarea
+    id: manual-installation
+    attributes:
+      label: Manual installation - system information
+      description: |
+        If you have installed Semaphore using the package or binary:
+        
+        Please share your operating system & -version!
+        > Command: `uname -a`
+        
+        What reverse proxy are you using?
+    validations:
+      required: false
+
+  - type: textarea
+    id: config
+    attributes:
+      label: Configuration
+      description: |
+        Please provide Semaphore configuration related to your problem - like:
+        * Config file options
+        * Environment variables
+        * WebUI configuration
+          * Task templates
+          * Inventories
+          * Environment
+          * Repositories
+          * ...
+
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional
+    attributes:
+      label: Additional information
+      description: |
+        Do you have additional information that could help troubleshoot the problem?
+
+    validations:
+      required: false

.github/ISSUE_TEMPLATES/question.yml

@@ -0,0 +1,45 @@
+---
+
+name: Question
+description: You have a question on how to use Semaphore
+title: "Question: "
+labels: ['question', 'triage']
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please make sure to go through these steps **before opening an issue**:
+        
+        - [ ] Read the [documentation](https://docs.ansible-semaphore.com/)
+          - [ ] Read the [troubleshooting guide](https://docs.ansible-semaphore.com/administration-guide/troubleshooting)
+          - [ ] Read the [documentation regarding manual installations](https://docs.ansible-semaphore.com/administration-guide/installation_manually) if you did install Semaphore that way
+
+        - [ ] Check if there are existing [issues](https://github.com/ansible-semaphore/semaphore/issues) or [discussions](https://github.com/ansible-semaphore/semaphore/discussions) regarding your topic
+
+  - type: textarea
+    id: question
+    attributes:
+      label: Question
+    validations:
+      required: true
+
+  - type: dropdown
+    id: related-to
+    attributes:
+      label: Related to
+      description: |
+        To what parts of Semaphore is the question related? (if any)
+
+      multiple: true
+      options:
+        - Web-Frontend (what users interact with)
+        - Web-Backend (APIs)
+        - Service (scheduled tasks, alerts)
+        - Ansible (task execution)
+        - Configuration
+        - Database
+        - Documentation
+        - Docker
+    validations:
+      required: false

.github/workflows/beta.yml

@@ -0,0 +1,75 @@
+name: Beta
+on:
+  push:
+    tags:
+      - v*-beta
+
+jobs:
+  pre-release:
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/setup-go@v3
+        with: { go-version: '1.21' }
+
+      - uses: actions/setup-node@v3
+        with: { node-version: '16' }
+
+      - run: go install github.com/go-task/task/v3/cmd/task@latest
+
+      - run: sudo apt update && sudo apt-get install rpm
+
+      - uses: actions/checkout@v3
+
+      - run: task deps
+
+      - run: |
+          echo ${{ secrets.GPG_KEY }} | tr " " "\n" | base64 -d | gpg --import --batch
+          gpg --sign -u "58A7 CC3D 8A9C A2E5 BB5C 141D 4064 23EA F814 63CA" --pinentry-mode loopback --yes --batch --passphrase "${{ secrets.GPG_PASS }}" --output unlock.sig --detach-sign README.md
+          rm -f unlock.sig
+
+      - run:  git reset --hard
+
+      - run: GITHUB_TOKEN=${{ secrets.GH_TOKEN }} task release:prod
+
+
+  deploy-beta:
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/setup-go@v3
+        with: { go-version: '1.21' }
+
+      - run: go install github.com/go-task/task/v3/cmd/task@latest
+
+      - uses: actions/checkout@v3
+
+      - run: context=prod task docker:test
+
+      - uses: docker/setup-qemu-action@v2
+
+      - uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASS }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./deployment/docker/prod/buildx.Dockerfile
+          push: true
+          tags: semaphoreui/semaphore:beta,semaphoreui/semaphore:${{ github.ref_name }}
+
+
+
+      - name: Build and push runner
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./deployment/docker/prod/runner.buildx.Dockerfile
+          push: true
+          tags: semaphoreui/runner:beta,semaphoreui/runner:${{ github.ref_name }}

.github/workflows/dev.yml

@@ -3,13 +3,15 @@ on:
   push:
     branches:
       - develop
+  pull_request:
+    branches: [develop]
 
 jobs:
   build-local:
     runs-on: [ubuntu-latest]
     steps:
       - uses: actions/setup-go@v3
-        with: { go-version: 1.18 }
+        with: { go-version: '1.21' }
 
       - uses: actions/setup-node@v3
         with: { node-version: '16' }
@@ -36,24 +38,6 @@ jobs:
           retention-days: 1
 
 
-#  test-golang:
-#    runs-on: [ubuntu-latest]
-#    needs: build-local
-#    steps:
-#      - uses: actions/setup-go@v3
-#        with: { go-version: 1.18 }
-#
-#      - run: go install github.com/go-task/task/v3/cmd/task@latest
-#
-#      - uses: actions/checkout@v3
-#
-#      - run: task deps:tools
-#      - run: task deps:be
-#      - run: task compile:be
-#      - run: task lint:be
-#      - run: task test
-
-
   test-db-migration:
     runs-on: [ubuntu-latest]
     needs: [build-local]
@@ -73,12 +57,7 @@ jobs:
         with:
           name: semaphore
 
-      - run: "cat > config.json <<EOF\n{\n\t\"postgres\": {\n\t\t\"host\": \"127.0.0.1:5432\"\
+      - run: sleep 5
-          ,\n\t\t\"options\":{\"sslmode\":\"disable\"}\
-          ,\n\t\t\"user\": \"root\",\n\t\t\"pass\": \"pwd\",\n\t\t\"name\": \"circle_test\"\
-          \n\t},\n\t\"dialect\": \"postgres\",\n\t\"email_alert\": false\n}\nEOF\n"
-
-      - run: chmod +x ./semaphore && ./semaphore migrate --config config.json
 
       - run: "cat > config.json <<EOF\n{\n\t\"mysql\": {\n\t\t\"host\": \"127.0.0.1:3306\"\
           ,\n\t\t\"user\": \"root\",\n\t\t\"pass\": \"\",\n\t\t\"name\": \"circle_test\"\
@@ -86,18 +65,24 @@ jobs:
 
       - run: chmod +x ./semaphore && ./semaphore migrate --config config.json
 
+      - run: "cat > config.json <<EOF\n{\n\t\"postgres\": {\n\t\t\"host\": \"127.0.0.1:5432\"\
+          ,\n\t\t\"options\":{\"sslmode\":\"disable\"}\
+          ,\n\t\t\"user\": \"root\",\n\t\t\"pass\": \"pwd\",\n\t\t\"name\": \"circle_test\"\
+          \n\t},\n\t\"dialect\": \"postgres\",\n\t\"email_alert\": false\n}\nEOF\n"
+
+      - run: chmod +x ./semaphore && ./semaphore migrate --config config.json
+
       - run: "cat > config.json <<EOF\n{\n\t\"bolt\": {\n\t\t\"host\": \"/tmp/database.bolt\"\
           \n\t},\n\t\"dialect\": \"bolt\",\n\t\"email_alert\": false\n}\nEOF\n"
 
       - run: chmod +x ./semaphore && ./semaphore migrate --config config.json
 
-
   test-integration:
     runs-on: [ubuntu-latest]
     needs: [test-db-migration]
     steps:
       - uses: actions/setup-go@v3
-        with: { go-version: 1.18 }
+        with: { go-version: '1.21' }
 
       - run: go install github.com/go-task/task/v3/cmd/task@latest
 
@@ -110,16 +95,15 @@ jobs:
   deploy-dev:
     runs-on: [ubuntu-latest]
     needs: [test-integration]
+    if: github.ref == 'refs/heads/develop'
     steps:
       - uses: actions/setup-go@v3
-        with: { go-version: 1.18 }
+        with: { go-version: '1.21' }
 
       - run: go install github.com/go-task/task/v3/cmd/task@latest
 
       - uses: actions/checkout@v3
 
-#      - run: context=prod task docker:test
-
       - uses: docker/setup-qemu-action@v2
 
       - uses: docker/setup-buildx-action@v2
@@ -139,17 +123,12 @@ jobs:
           push: true
           tags: semaphoreui/semaphore:develop
 
+      - name: Build and push runner
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./deployment/docker/prod/runner.buildx.Dockerfile
+          push: true
+          tags: semaphoreui/runner:develop
 
-
-
-  # test-docker:
-  #   runs-on: [ubuntu-latest]
-  #   steps:
-  #     - uses: actions/setup-go@v3
-  #       with: { go-version: 1.18 }
-
-  #     - run: go install github.com/go-task/task/v3/cmd/task@latest
-
-  #     - uses: actions/checkout@v3
-
-  #     - run: context=prod task docker:test

.github/workflows/release.yml

@@ -2,14 +2,14 @@ name: Release
 on:
   push:
     tags:
-      - v*
+      - 'v[0-9]+.[0-9]+.[0-9]+'
 
 jobs:
   release:
     runs-on: [ubuntu-latest]
     steps:
       - uses: actions/setup-go@v3
-        with: { go-version: 1.18 }
+        with: { go-version: '1.21' }
 
       - uses: actions/setup-node@v3
         with: { node-version: '16' }
@@ -36,7 +36,7 @@ jobs:
     runs-on: [ubuntu-latest]
     steps:
       - uses: actions/setup-go@v3
-        with: { go-version: 1.18 }
+        with: { go-version: '1.21' }
 
       - run: go install github.com/go-task/task/v3/cmd/task@latest
 
@@ -61,4 +61,15 @@ jobs:
           platforms: linux/amd64,linux/arm64
           file: ./deployment/docker/prod/buildx.Dockerfile
           push: true
-          tags: semaphoreui/semaphore:latest,semaphoreui/semaphore:${{ github.ref_name }}
+          tags: semaphoreui/semaphore:latest,semaphoreui/semaphore:${{ github.ref_name }}
+
+
+
+      - name: Build and push runner
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./deployment/docker/prod/runner.buildx.Dockerfile
+          push: true
+          tags: semaphoreui/runner:latest,semaphoreui/runner:${{ github.ref_name }}

.github/workflows/test.yml

@@ -1,18 +0,0 @@
-name: Test
-on:
-  push:
-    branches:
-    - test
-
-jobs:
-  test-integration:
-    runs-on: [ubuntu-latest]
-    steps:
-    - uses: actions/setup-go@v3
-      with: { go-version: 1.18 }
-
-    - run: go install github.com/go-task/task/v3/cmd/task@latest
-
-    - uses: actions/checkout@v3
-
-    - run: context=ci task dc:up

.gitignore

@@ -5,7 +5,7 @@ web/public/css/*.*
 web/public/html/**/*.*
 web/public/fonts/*.*
 web/.nyc_output
-web/dist/**/*
+api/public/**/*
 /config.json
 /.dredd/config.json
 /database.boltdb
@@ -18,7 +18,6 @@ node_modules/
 /semaphore.iml
 /bin/
 
-*-packr.go
 util/version.go
 /vendor/
 /coverage.out
@@ -30,4 +29,4 @@ util/version.go
 
 .vscode
 __debug_bin*
-.task/
+.task/

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Pull Requests
+## Pull Requests
 
 When creating a pull-request you should:
 
@@ -8,7 +8,7 @@ When creating a pull-request you should:
 - __Update api documentation:__ If your pull-request adding/modifying an API request, make sure you update the swagger documentation (`api-docs.yml`)
 - __Run Api Tests:__ If your pull request modifies the API make sure you run the integration tests using dredd.
 
-# Installation in a development environment
+## Installation in a development environment
 
 - Check out the `develop` branch
 - [Install Go](https://golang.org/doc/install). Go must be >= v1.10 for all the tools we use to work
@@ -59,14 +59,32 @@ Dredd is used for API integration tests, if you alter the API in any way you mus
 matches the responses.
 
 As Dredd and the application database config may differ it expects it's own config.json in the .dredd folder.
-The most basic configuration for this using a local docker container to run the database would be
+
-```json
+### How to run Dredd tests locally
-{
+
-	"mysql": {
+1) Build Dredd hooks:
-		"host": "0.0.0.0:3306",
+    ````bash
-		"user": "semaphore",
+    task compile:api:hooks
-		"pass": "semaphore",
+    ```
-		"name": "semaphore"
+2) Install Dredd globally
-	}
+    ```bash
-}
+    npm install -g dredd
-```
+    ```
+3) Create `./dredd/config.json` for Dredd. It must contain database connection same as used in Semaphore server.
+   You can use any supported database dialect for tests. For example BoltDB.
+    ```json
+   {
+        "bolt": {
+            "host": "/tmp/database.boltdb"
+        },
+        "dialect": "bolt"
+    }
+    ```
+4) Start Semaphore server (add `--config` option if required):
+    ````bash
+    ./bin/semaphore server
+    ```
+5) Start Dredd tests
+    ```
+    dredd --config ./.dredd/dredd.local.yml
+    ```

README.md

@@ -1,14 +1,11 @@
 # Ansible Semaphore
 
-[![Twitter](https://img.shields.io/twitter/follow/semaphoreui?style=social&logo=twitter)](https://twitter.com/semaphoreui)
 [![semaphore](https://snapcraft.io/semaphore/badge.svg)](https://snapcraft.io/semaphore)
-[![StackShare](https://img.shields.io/badge/tech-stack-008ff9)](https://stackshare.io/ansible-semaphore)
 [![Join the chat at https://gitter.im/AnsibleSemaphore/semaphore](https://img.shields.io/gitter/room/AnsibleSemaphore/semaphore?logo=gitter)](https://gitter.im/AnsibleSemaphore/semaphore?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-<!-- [![Release](https://img.shields.io/github/v/release/ansible-semaphore/semaphore.svg)](https://stackshare.io/ansible-semaphore) -->
+[![Twitter](https://img.shields.io/twitter/follow/semaphoreui?style=social&logo=twitter)](https://twitter.com/semaphoreui)
-<!-- [![Godoc Reference](https://pkg.go.dev/badge/github.com/ansible-semaphore/semaphore?utm_source=godoc)](https://godoc.org/github.com/ansible-semaphore/semaphore) -->
+
-<!-- [![Codacy Badge](https://api.codacy.com/project/badge/Grade/89e0129c6ba64fe2b1ebe983f72a4eff)](https://www.codacy.com/app/ansible-semaphore/semaphore?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=ansible-semaphore/semaphore&amp;utm_campaign=Badge_Grade)
+[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/fiftin)
-[![Codacy Badge](https://api.codacy.com/project/badge/Coverage/89e0129c6ba64fe2b1ebe983f72a4eff)](https://www.codacy.com/app/ansible-semaphore/semaphore?utm_source=github.com&utm_medium=referral&utm_content=ansible-semaphore/semaphore&utm_campaign=Badge_Coverage) -->
 
 Ansible Semaphore is a modern UI for Ansible. It lets you easily run Ansible playbooks, get notifications about fails, control access to deployment system.
 
@@ -16,27 +13,10 @@ If your project has grown and deploying from the terminal is no longer for you t
 
 ![responsive-ui-phone1](https://user-images.githubusercontent.com/914224/134777345-8789d9e4-ff0d-439c-b80e-ddc56b74fcee.png)
 
-<!--
-![image](https://user-images.githubusercontent.com/914224/134411082-48235676-06d2-4d4b-b674-4ffe1e8d0d0d.png)
-
-![semaphore](https://user-images.githubusercontent.com/914224/125253358-c214ed80-e312-11eb-952e-d96a1eba93f6.png)
--->
-
-
-<!--
-- [Releases](https://github.com/ansible-semaphore/semaphore/releases)
-- [Installation](https://docs.ansible-semaphore.com/administration-guide/installation)
-- [Docker Hub](https://hub.docker.com/r/semaphoreui/semaphore/)
-- [Contribution](https://github.com/ansible-semaphore/semaphore/blob/develop/CONTRIBUTING.md)
-- [Troubleshooting](https://github.com/ansible-semaphore/semaphore/wiki/Troubleshooting)
-- [Roadmap](https://github.com/ansible-semaphore/semaphore/projects)
-- [UI Walkthrough](https://blog.strangeman.info/ansible/2017/08/05/semaphore-ui-guide.html) (external blog)
--->
-
 ## Installation
 
 ### Full documentation
-https://docs.ansible-semaphore.com/administration-guide/installation
+https://docs.semui.co/administration-guide/installation
 
 ### Snap
 
@@ -46,7 +26,9 @@ sudo semaphore user add --admin --name "Your Name" --login your_login --email yo
 ```
 [![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/semaphore)
 
-### Docker 
+### Docker
+
+https://hub.docker.com/r/semaphoreui/semaphore
 
 `docker-compose.yml` for minimal configuration:
 
@@ -62,24 +44,26 @@ services:
       SEMAPHORE_ADMIN_NAME: admin
       SEMAPHORE_ADMIN_EMAIL: admin@localhost
       SEMAPHORE_ADMIN: admin
+      TZ: Europe/Berlin
     volumes:
       - /path/to/data/home:/etc/semaphore # config.json location
       - /path/to/data/lib:/var/lib/semaphore # database.boltdb location (Not required if using mysql or postgres)
 ```
-https://hub.docker.com/r/semaphoreui/semaphore
 
 ## Demo
 
-You can test latest version of Semaphore on https://demo.ansible-semaphore.com.
+You can test latest version of Semaphore on https://demo.semui.co.
 
 ## Docs
 
-Admin and user docs: https://docs.ansible-semaphore.com
+Admin and user docs: https://docs.semui.co.
 
-API description: https://ansible-semaphore.com/api-docs/
+API description: https://semui.co/api-docs/.
 
 ## Contributing
 
+If you want to write an article about Ansible or Semaphore, contact [@fiftin](https://github.com/fiftin) and we will place your article in our [Blog](https://semui.co/blog/) with link to your profile.
+
 PR's & UX reviews are welcome!
 
 Please follow the [contribution](https://github.com/ansible-semaphore/semaphore/blob/develop/CONTRIBUTING.md) guide. Any questions, please open an issue.
@@ -93,9 +77,6 @@ All releases after 2.5.1 are signed with the gpg public key
 
 If you like Ansible Semaphore, you can support the project development on [Ko-fi](https://ko-fi.com/fiftin).
 
-[<img src="https://user-images.githubusercontent.com/914224/203517453-4febf7f6-debb-4be9-b6a2-a3b19f5d9f9a.png">](https://ko-fi.com/fiftin)
-
-
 ## License
 
 MIT License

Taskfile.yml

@@ -3,7 +3,7 @@
 #
 # Tasks without a `desc:` field are intended mainly to be called
 # internally by other tasks and therefore are not listed when running `task` or `task -l`
-version: '2'
+version: '3'
 
 vars:
   docker_namespace: semaphoreui
@@ -56,7 +56,6 @@ tasks:
       GORELEASER_VERSION: "0.183.0"
       GOLINTER_VERSION: "1.46.2"
     cmds:
-      - go install github.com/gobuffalo/packr/...@v1.10.4
       - go install github.com/snikch/goodman/cmd/goodman@latest
       - go install github.com/go-swagger/go-swagger/cmd/swagger@v0.29.0
       - '{{ if ne OS "windows" }} sh -c "curl -L https://github.com/goreleaser/goreleaser/releases/download/v{{ .GORELEASER_VERSION }}/goreleaser_$(uname -s)_$(uname -m).tar.gz | tar -xz -C $(go env GOPATH)/bin goreleaser"{{ else }} {{ end }}'
@@ -84,25 +83,17 @@ tasks:
       - babel.config.js
       - vue.config.js
     generates:
-      - dist/css/*.css
+      - ../api/public/css/*.css
-      - dist/js/*.js
+      - ../api/public/js/*.js
-      - dist/index.html
+      - ../api/public/index.html
-      - dist/favicon.ico
+      - ../api/public/favicon.ico
     cmds:
       - npm run build
 
   compile:be:
-    desc: Runs Packr for static assets
+    desc: Generate the version
-    sources:
-      - web/dist/*
-      - db/migrations/*
-    generates:
-      - db/db-packr.go
-      - api/api-packr.go
     cmds:
-      - mkdir -p web/dist
       - go run util/version_gen/generator.go {{ if .TAG }}{{ .TAG }}{{ else }}{{ if .SEMAPHORE_VERSION }}{{ .SEMAPHORE_VERSION }}{{ else }}{{ .BRANCH }}-{{ .SHA }}-{{ .TIMESTAMP }}{{ if .DIRTY }}-dirty{{ end }}{{ end }}{{end}}
-      - packr
     vars:
       TAG:
         sh: git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null | sed -n 's/^\([^^~]\{1,\}\)\(\^0\)\{0,1\}$/\1/p'
@@ -148,7 +139,7 @@ tasks:
   lint:be:
     # --errors
     cmds:
-      - golangci-lint run --skip-files "\w*(-packr.go)" --disable goconst --timeout 240s ./...
+      - golangci-lint run --disable goconst --timeout 240s ./...
 
   test:
     cmds:

Taskfile_windows.yml

@@ -2,7 +2,6 @@ version: '2'
 tasks:
   compile:be:
     cmds:
-      - packr
       - go run util/version_gen/generator.go 1
   build:local:
     dir: cli

api/auth.go

@@ -1,7 +1,7 @@
 package api
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/ansible-semaphore/semaphore/util"
@@ -90,15 +90,6 @@ func authenticationHandler(w http.ResponseWriter, r *http.Request) bool {
 		return false
 	}
 
-	if util.Config.DemoMode {
-		if !user.Admin && r.Method != "GET" &&
-			!strings.HasSuffix(r.URL.Path, "/tasks") &&
-			!strings.HasSuffix(r.URL.Path, "/stop") {
-			w.WriteHeader(http.StatusUnauthorized)
-			return false
-		}
-	}
-
 	context.Set(r, "user", &user)
 	return true
 }
@@ -119,7 +110,7 @@ func authenticationWithStore(next http.Handler) http.Handler {
 		store := helpers.Store(r)
 
 		var ok bool
-		
+
 		db.StoreSession(store, r.URL.String(), func() {
 			ok = authenticationHandler(w, r)
 		})

api/events.go

@@ -8,7 +8,7 @@ import (
 	"github.com/gorilla/context"
 )
 
-//nolint: gocyclo
+// nolint: gocyclo
 func getEvents(w http.ResponseWriter, r *http.Request, limit int) {
 	user := context.Get(r, "user").(*db.User)
 	projectObj, exists := context.GetOk(r, "project")
@@ -19,7 +19,9 @@ func getEvents(w http.ResponseWriter, r *http.Request, limit int) {
 	if exists {
 		project := projectObj.(db.Project)
 
-		_, err = helpers.Store(r).GetProjectUser(project.ID, user.ID)
+		if !user.Admin { // check permissions to view events
+			_, err = helpers.Store(r).GetProjectUser(project.ID, user.ID)
+		}
 
 		if err != nil {
 			helpers.WriteError(w, err)

api/helpers/helpers.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/gorilla/context"
 
 	"github.com/ansible-semaphore/semaphore/db"
@@ -48,10 +48,10 @@ func GetIntParam(name string, w http.ResponseWriter, r *http.Request) (int, erro
 	return intParam, nil
 }
 
-//H just a string-to-anything map
+// H just a string-to-anything map
 type H map[string]interface{}
 
-//Bind decodes json into object
+// Bind decodes json into object
 func Bind(w http.ResponseWriter, r *http.Request, out interface{}) bool {
 	err := json.NewDecoder(r.Body).Decode(out)
 	if err != nil {
@@ -61,7 +61,7 @@ func Bind(w http.ResponseWriter, r *http.Request, out interface{}) bool {
 	return err == nil
 }
 
-//WriteJSON writes object as JSON
+// WriteJSON writes object as JSON
 func WriteJSON(w http.ResponseWriter, code int, out interface{}) {
 	w.Header().Set("content-type", "application/json")
 	w.WriteHeader(code)

api/integration.go

@@ -0,0 +1,225 @@
+package api
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/json"
+	"fmt"
+	"github.com/ansible-semaphore/semaphore/util"
+	"github.com/gorilla/context"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/ansible-semaphore/semaphore/api/helpers"
+	"github.com/ansible-semaphore/semaphore/db"
+	log "github.com/sirupsen/logrus"
+	jsonq "github.com/thedevsaddam/gojsonq/v2"
+)
+
+// IsValidPayload checks if the github payload's hash fits with
+// the hash computed by GitHub sent as a header
+func IsValidPayload(secret, headerHash string, payload []byte) bool {
+	hash := HashPayload(secret, payload)
+	return hmac.Equal(
+		[]byte(hash),
+		[]byte(headerHash),
+	)
+}
+
+// HashPayload computes the hash of payload's body according to the webhook's secret token
+// see https://developer.github.com/webhooks/securing/#validating-payloads-from-github
+// returning the hash as a hexadecimal string
+func HashPayload(secret string, playloadBody []byte) string {
+	hm := hmac.New(sha1.New, []byte(secret))
+	hm.Write(playloadBody)
+	sum := hm.Sum(nil)
+	return fmt.Sprintf("%x", sum)
+}
+
+func ReceiveIntegration(w http.ResponseWriter, r *http.Request) {
+
+	if !util.Config.IntegrationsEnable {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	log.Info(fmt.Sprintf("Receiving Integration from: %s", r.RemoteAddr))
+
+	var err error
+
+	//project := context.Get(r, "project").(db.Project)
+	projectId, err := helpers.GetIntParam("project_id", w, r)
+	integration := context.Get(r, "integration").(db.Integration)
+
+	switch integration.AuthMethod {
+	case db.IntegrationAuthHmac:
+		var payload []byte
+		_, err = r.Body.Read(payload)
+		if err != nil {
+			log.Error(err)
+			return
+		}
+
+		if IsValidPayload(integration.AuthSecret.LoginPassword.Password, r.Header.Get(integration.AuthHeader), payload) {
+			log.Error(err)
+			return
+		}
+	case db.IntegrationAuthToken:
+		if integration.AuthSecret.LoginPassword.Password != r.Header.Get(integration.AuthHeader) {
+			log.Error("Invalid verification token")
+			return
+		}
+	case db.IntegrationAuthNone:
+	default:
+		log.Error("Unknown verification method: " + integration.AuthMethod)
+		return
+	}
+
+	var matchers []db.IntegrationMatcher
+	matchers, err = helpers.Store(r).GetIntegrationMatchers(projectId, db.RetrieveQueryParams{}, integration.ID)
+	if err != nil {
+		log.Error(err)
+	}
+	var matched = false
+
+	for _, matcher := range matchers {
+		if Match(matcher, r) {
+			matched = true
+			continue
+		} else {
+			matched = false
+			break
+		}
+	}
+
+	if !matched {
+		w.WriteHeader(http.StatusNoContent)
+		return
+	}
+
+	RunIntegration(integration, r)
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func Match(matcher db.IntegrationMatcher, r *http.Request) (matched bool) {
+
+	switch matcher.MatchType {
+	case db.IntegrationMatchHeader:
+		var header_value = r.Header.Get(matcher.Key)
+		return MatchCompare(header_value, matcher.Method, matcher.Value)
+	case db.IntegrationMatchBody:
+		bodyBytes, err := io.ReadAll(r.Body)
+
+		if err != nil {
+			log.Fatalln(err)
+			return false
+		}
+		var body = string(bodyBytes)
+		switch matcher.BodyDataType {
+		case db.IntegrationBodyDataJSON:
+			var jsonBytes bytes.Buffer
+			jsonq.New().FromString(body).From(matcher.Key).Writer(&jsonBytes)
+			var jsonString = jsonBytes.String()
+			if err != nil {
+				log.Error(fmt.Sprintf("Failed to marshal JSON contents of body. %v", err))
+			}
+			return MatchCompare(jsonString, matcher.Method, matcher.Value)
+		case db.IntegrationBodyDataString:
+			return MatchCompare(body, matcher.Method, matcher.Value)
+		case db.IntegrationBodyDataXML:
+			// XXX: TBI
+			return false
+		}
+	}
+
+	return false
+}
+
+func MatchCompare(value string, method db.IntegrationMatchMethodType, expected string) bool {
+	switch method {
+	case db.IntegrationMatchMethodEquals:
+		return value == expected
+	case db.IntegrationMatchMethodUnEquals:
+		return value != expected
+	case db.IntegrationMatchMethodContains:
+		return strings.Contains(value, expected)
+	default:
+		return false
+	}
+}
+
+func RunIntegration(integration db.Integration, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+
+	var extractValues = make([]db.IntegrationExtractValue, 0)
+
+	extractValuesForExtractor, err2 := helpers.Store(r).GetIntegrationExtractValues(project.ID, db.RetrieveQueryParams{}, integration.ID)
+	if err2 != nil {
+		log.Error(err2)
+		return
+	}
+	extractValues = append(extractValues, extractValuesForExtractor...)
+
+	var extractedResults = Extract(extractValues, r)
+
+	// XXX: LOG AN EVENT HERE
+	environmentJSONBytes, err := json.Marshal(extractedResults)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	var environmentJSONString = string(environmentJSONBytes)
+	var taskDefinition = db.Task{
+		TemplateID:  integration.TemplateID,
+		ProjectID:   integration.ProjectID,
+		Debug:       true,
+		Environment: environmentJSONString,
+	}
+
+	var user db.User
+	user, err = helpers.Store(r).GetUser(1)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	_, err = helpers.TaskPool(r).AddTask(taskDefinition, &user.ID, integration.ProjectID)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+}
+
+func Extract(extractValues []db.IntegrationExtractValue, r *http.Request) (result map[string]string) {
+	result = make(map[string]string)
+
+	for _, extractValue := range extractValues {
+		switch extractValue.ValueSource {
+		case db.IntegrationExtractHeaderValue:
+			result[extractValue.Variable] = r.Header.Get(extractValue.Key)
+		case db.IntegrationExtractBodyValue:
+			bodyBytes, err := io.ReadAll(r.Body)
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+			var body = string(bodyBytes)
+
+			switch extractValue.BodyDataType {
+			case db.IntegrationBodyDataJSON:
+				var jsonBytes bytes.Buffer
+				jsonq.New().FromString(body).From(extractValue.Key).Writer(&jsonBytes)
+				result[extractValue.Variable] = jsonBytes.String()
+			case db.IntegrationBodyDataString:
+				result[extractValue.Variable] = body
+			case db.IntegrationBodyDataXML:
+				// XXX: TBI
+			}
+		}
+	}
+	return
+}

api/login.go

@@ -1,21 +1,31 @@
 package api
 
 import (
+	"context"
 	"crypto/tls"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"math/rand"
 	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
 	"golang.org/x/crypto/bcrypt"
+	"golang.org/x/oauth2"
 
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/go-ldap/ldap/v3"
+	"github.com/gorilla/mux"
 
-	log "github.com/Sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/util"
+	log "github.com/sirupsen/logrus"
 )
 
 func tryFindLDAPUser(username, password string) (*db.User, error) {
@@ -71,15 +81,6 @@ func tryFindLDAPUser(username, password string) (*db.User, error) {
 		return nil, err
 	}
 
-	// Ensure authentication and verify itself with whoami operation
-	var res *ldap.WhoAmIResult
-	if res, err = l.WhoAmI(nil); err != nil {
-		return nil, err
-	}
-	if len(res.AuthzID) <= 0 {
-		return nil, fmt.Errorf("error while doing whoami operation")
-	}
-
 	// Second time bind as read only user
 	if err = l.Bind(util.Config.LdapBindDN, util.Config.LdapBindPassword); err != nil {
 		return nil, err
@@ -192,8 +193,47 @@ func loginByLDAP(store db.Store, ldapUser db.User) (user db.User, err error) {
 	return
 }
 
+type loginMetadataOidcProvider struct {
+	ID    string `json:"id"`
+	Name  string `json:"name"`
+	Color string `json:"color"`
+	Icon  string `json:"icon"`
+}
+
+type loginMetadata struct {
+	OidcProviders     []loginMetadataOidcProvider `json:"oidc_providers"`
+	LoginWithPassword bool                        `json:"login_with_password"`
+}
+
 // nolint: gocyclo
 func login(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "GET" {
+		config := &loginMetadata{
+			OidcProviders:     make([]loginMetadataOidcProvider, len(util.Config.OidcProviders)),
+			LoginWithPassword: !util.Config.PasswordLoginDisable,
+		}
+		i := 0
+
+		for k, v := range util.Config.OidcProviders {
+			config.OidcProviders[i] = loginMetadataOidcProvider{
+				ID:    k,
+				Name:  v.DisplayName,
+				Color: v.Color,
+				Icon:  v.Icon,
+			}
+			i++
+		}
+
+		sort.Slice(config.OidcProviders, func(i, j int) bool {
+			a := util.Config.OidcProviders[config.OidcProviders[i].ID]
+			b := util.Config.OidcProviders[config.OidcProviders[j].ID]
+			return a.Order < b.Order
+		})
+
+		helpers.WriteJSON(w, http.StatusOK, config)
+		return
+	}
+
 	var login struct {
 		Auth     string `json:"auth" binding:"required"`
 		Password string `json:"password" binding:"required"`
@@ -264,3 +304,314 @@ func logout(w http.ResponseWriter, r *http.Request) {
 
 	w.WriteHeader(http.StatusNoContent)
 }
+
+func getOidcProvider(id string, ctx context.Context, redirectPath string) (*oidc.Provider, *oauth2.Config, error) {
+	provider, ok := util.Config.OidcProviders[id]
+	if !ok {
+		return nil, nil, fmt.Errorf("No such provider: %s", id)
+	}
+	config := oidc.ProviderConfig{
+		IssuerURL:   provider.Endpoint.IssuerURL,
+		AuthURL:     provider.Endpoint.AuthURL,
+		TokenURL:    provider.Endpoint.TokenURL,
+		UserInfoURL: provider.Endpoint.UserInfoURL,
+		JWKSURL:     provider.Endpoint.JWKSURL,
+		Algorithms:  provider.Endpoint.Algorithms,
+	}
+	oidcProvider := config.NewProvider(ctx)
+	var err error
+	if len(provider.AutoDiscovery) > 0 {
+		oidcProvider, err = oidc.NewProvider(ctx, provider.AutoDiscovery)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	clientID := provider.ClientID
+	if provider.ClientIDFile != "" {
+		if clientID, err = getSecretFromFile(provider.ClientIDFile); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	clientSecret := provider.ClientSecret
+	if provider.ClientSecretFile != "" {
+		if clientSecret, err = getSecretFromFile(provider.ClientSecretFile); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	if redirectPath != "" {
+		if !strings.HasPrefix(redirectPath, "/") {
+			redirectPath = "/" + redirectPath
+		}
+
+		providerUrl, err := url.Parse(provider.RedirectURL)
+
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if redirectPath == providerUrl.Path {
+			redirectPath = ""
+		} else if strings.HasPrefix(redirectPath, providerUrl.Path+"/") {
+			redirectPath = redirectPath[len(providerUrl.Path):]
+		}
+	}
+
+	oauthConfig := oauth2.Config{
+		Endpoint:     oidcProvider.Endpoint(),
+		ClientID:     clientID,
+		ClientSecret: clientSecret,
+		RedirectURL:  provider.RedirectURL + redirectPath,
+		Scopes:       provider.Scopes,
+	}
+	if len(oauthConfig.RedirectURL) == 0 {
+		rurl, err := url.JoinPath(util.Config.WebHost, "api/auth/oidc", id, "redirect")
+		if err != nil {
+			return nil, nil, err
+		}
+
+		oauthConfig.RedirectURL = rurl
+
+		if rurl != redirectPath {
+			oauthConfig.RedirectURL += redirectPath
+		}
+	}
+	if len(oauthConfig.Scopes) == 0 {
+		oauthConfig.Scopes = []string{"openid", "profile", "email"}
+	}
+	return oidcProvider, &oauthConfig, nil
+}
+
+func oidcLogin(w http.ResponseWriter, r *http.Request) {
+	pid := mux.Vars(r)["provider"]
+	ctx := context.Background()
+
+	redirectPath := ""
+
+	if r.URL.Query()["redirect"] != nil {
+		// TODO: validate path
+		redirectPath = r.URL.Query()["redirect"][0]
+	}
+
+	_, oauth, err := getOidcProvider(pid, ctx, redirectPath)
+	if err != nil {
+		log.Error(err.Error())
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+	state := generateStateOauthCookie(w)
+	u := oauth.AuthCodeURL(state)
+	http.Redirect(w, r, u, http.StatusTemporaryRedirect)
+}
+
+func generateStateOauthCookie(w http.ResponseWriter) string {
+	expiration := time.Now().Add(365 * 24 * time.Hour)
+
+	b := make([]byte, 16)
+	rand.Read(b)
+	oauthState := base64.URLEncoding.EncodeToString(b)
+	cookie := http.Cookie{Name: "oauthstate", Value: oauthState, Expires: expiration}
+	http.SetCookie(w, &cookie)
+
+	return oauthState
+}
+
+type oidcClaimResult struct {
+	username string
+	name     string
+	email    string
+}
+
+func parseClaims(claims map[string]interface{}, provider util.OidcProvider) (res oidcClaimResult, err error) {
+	var ok bool
+
+	res.email, ok = claims[provider.EmailClaim].(string)
+	if !ok {
+
+		var username string
+
+		if provider.EmailSuffix == "" {
+			err = fmt.Errorf("claim '%s' missing from id_token or not a string", provider.EmailClaim)
+			return
+		}
+
+		switch claims[provider.UsernameClaim].(type) {
+		case float64:
+			username = strconv.FormatFloat(claims[provider.UsernameClaim].(float64), 'f', -1, 64)
+		case string:
+			username = claims[provider.UsernameClaim].(string)
+		default:
+			err = fmt.Errorf("claim '%s' missing from id_token or not a string or an number", provider.UsernameClaim)
+			b, _ := json.MarshalIndent(claims, "", "  ")
+			fmt.Print(string(b))
+			return
+		}
+
+		res.email = username + "@" + provider.EmailSuffix
+	}
+
+	res.username = getRandomUsername()
+
+	res.name, ok = claims[provider.NameClaim].(string)
+	if !ok || res.name == "" {
+		res.name = getRandomProfileName()
+	}
+
+	return
+}
+
+func claimOidcUserInfo(userInfo *oidc.UserInfo, provider util.OidcProvider) (res oidcClaimResult, err error) {
+
+	claims := make(map[string]interface{})
+	if err = userInfo.Claims(&claims); err != nil {
+		return
+	}
+
+	return parseClaims(claims, provider)
+}
+
+func claimOidcToken(idToken *oidc.IDToken, provider util.OidcProvider) (res oidcClaimResult, err error) {
+
+	claims := make(map[string]interface{})
+	if err = idToken.Claims(&claims); err != nil {
+		return
+	}
+
+	return parseClaims(claims, provider)
+}
+
+func getRandomUsername() string {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
+	result := ""
+	for i := 0; i < 16; i++ {
+		index := r.Intn(len(chars))
+		result += chars[index : index+1]
+	}
+	return result
+}
+
+func getRandomProfileName() string {
+	return "Anonymous"
+}
+
+func getSecretFromFile(source string) (string, error) {
+	content, err := os.ReadFile(source)
+
+	if err != nil {
+		return "", err
+	}
+
+	return string(content), nil
+}
+
+func oidcRedirect(w http.ResponseWriter, r *http.Request) {
+	pid := mux.Vars(r)["provider"]
+	oauthState, err := r.Cookie("oauthstate")
+	if err != nil {
+		log.Error(err.Error())
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+
+	if r.FormValue("state") != oauthState.Value {
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+
+	ctx := context.Background()
+
+	_oidc, oauth, err := getOidcProvider(pid, ctx, r.URL.Path)
+	if err != nil {
+		log.Error(err.Error())
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+
+	provider, ok := util.Config.OidcProviders[pid]
+	if !ok {
+		log.Error(fmt.Errorf("no such provider: %s", pid))
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+
+	verifier := _oidc.Verifier(&oidc.Config{ClientID: oauth.ClientID})
+
+	code := r.URL.Query().Get("code")
+
+	oauth2Token, err := oauth.Exchange(ctx, code)
+	if err != nil {
+		log.Error(err.Error())
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+
+	var claims oidcClaimResult
+
+	// Extract the ID Token from OAuth2 token.
+	rawIDToken, ok := oauth2Token.Extra("id_token").(string)
+
+	if ok && rawIDToken != "" {
+		var idToken *oidc.IDToken
+		// Parse and verify ID Token payload.
+		idToken, err = verifier.Verify(ctx, rawIDToken)
+
+		if err == nil {
+			claims, err = claimOidcToken(idToken, provider)
+		}
+	} else {
+		var userInfo *oidc.UserInfo
+		userInfo, err = _oidc.UserInfo(ctx, oauth2.StaticTokenSource(oauth2Token))
+
+		if err == nil {
+
+			if userInfo.Email == "" {
+				claims, err = claimOidcUserInfo(userInfo, provider)
+			} else {
+				claims.email = userInfo.Email
+				claims.name = userInfo.Profile
+			}
+		}
+
+		claims.username = getRandomUsername()
+		if userInfo.Profile == "" {
+			claims.name = getRandomProfileName()
+		}
+	}
+
+	if err != nil {
+		log.Error(err.Error())
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+
+	user, err := helpers.Store(r).GetUserByLoginOrEmail("", claims.email) // ignore username because it creates a lot of problems
+	if err != nil {
+		user = db.User{
+			Username: claims.username,
+			Name:     claims.name,
+			Email:    claims.email,
+			External: true,
+		}
+		user, err = helpers.Store(r).CreateUserWithoutPassword(user)
+		if err != nil {
+			log.Error(err.Error())
+			http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+			return
+		}
+	}
+
+	if !user.External {
+		log.Error(fmt.Errorf("OIDC user '%s' conflicts with local user", user.Username))
+		http.Redirect(w, r, "/auth/login", http.StatusTemporaryRedirect)
+		return
+	}
+
+	createSession(w, r, user)
+
+	redirectPath := mux.Vars(r)["redirect_path"]
+
+	http.Redirect(w, r, "/"+redirectPath, http.StatusTemporaryRedirect)
+}

api/projects/backupRestore.go

@@ -0,0 +1,48 @@
+package projects
+
+import (
+	"net/http"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/ansible-semaphore/semaphore/api/helpers"
+	"github.com/ansible-semaphore/semaphore/db"
+	projectService "github.com/ansible-semaphore/semaphore/services/project"
+	"github.com/gorilla/context"
+)
+
+func GetBackup(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+
+	store := helpers.Store(r)
+
+	backup, err := projectService.GetBackup(project.ID, store)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+	helpers.WriteJSON(w, http.StatusOK, backup)
+}
+
+func Restore(w http.ResponseWriter, r *http.Request) {
+	var backup projectService.BackupFormat
+	var p *db.Project
+	var err error
+
+	if !helpers.Bind(w, r, &backup) {
+		helpers.WriteJSON(w, http.StatusBadRequest, backup)
+		return
+	}
+	store := helpers.Store(r)
+	if err = backup.Verify(); err != nil {
+		log.Error(err)
+		helpers.WriteError(w, (err))
+		return
+	}
+	if p, err = backup.Restore(store); err != nil {
+		log.Error(err)
+		helpers.WriteError(w, (err))
+		return
+	}
+	helpers.WriteJSON(w, http.StatusOK, p)
+}

api/projects/environment.go

@@ -1,7 +1,7 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"net/http"

api/projects/integration.go

@@ -0,0 +1,173 @@
+package projects
+
+import (
+	"fmt"
+	"net/http"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/ansible-semaphore/semaphore/api/helpers"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/context"
+)
+
+func IntegrationMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		integrationId, err := helpers.GetIntParam("integration_id", w, r)
+		projectId, err := helpers.GetIntParam("project_id", w, r)
+
+		if err != nil {
+			helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+				"error": "Invalid integration ID",
+			})
+			return
+		}
+
+		integration, err := helpers.Store(r).GetIntegration(projectId, integrationId)
+
+		if err != nil {
+			helpers.WriteError(w, err)
+			return
+		}
+
+		context.Set(r, "integration", integration)
+		next.ServeHTTP(w, r)
+	})
+}
+
+func GetIntegration(w http.ResponseWriter, r *http.Request) {
+	integration := context.Get(r, "integration").(db.Integration)
+	helpers.WriteJSON(w, http.StatusOK, integration)
+}
+
+func GetIntegrations(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	integrations, err := helpers.Store(r).GetIntegrations(project.ID, helpers.QueryParams(r.URL))
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, integrations)
+}
+
+func GetIntegrationRefs(w http.ResponseWriter, r *http.Request) {
+	integration_id, err := helpers.GetIntParam("integration_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Integration ID",
+		})
+		return
+	}
+
+	project := context.Get(r, "project").(db.Project)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+	refs, err := helpers.Store(r).GetIntegrationRefs(project.ID, integration_id)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, refs)
+}
+
+func AddIntegration(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	var integration db.Integration
+	log.Info(fmt.Sprintf("Found Project: %v", project.ID))
+
+	if !helpers.Bind(w, r, &integration) {
+		log.Info("Failed to bind for integration uploads")
+
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Project ID in body and URL must be the same",
+		})
+		return
+	}
+
+	if integration.ProjectID != project.ID {
+		log.Error(fmt.Sprintf("Project ID in body and URL must be the same: %v vs. %v", integration.ProjectID, project.ID))
+
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Project ID in body and URL must be the same",
+		})
+		return
+	}
+	err := integration.Validate()
+	if err != nil {
+		log.Error(err)
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": err.Error(),
+		})
+		return
+	}
+
+	newIntegration, errIntegration := helpers.Store(r).CreateIntegration(integration)
+
+	if errIntegration != nil {
+		log.Error(errIntegration)
+		helpers.WriteError(w, errIntegration)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusCreated, newIntegration)
+}
+
+func UpdateIntegration(w http.ResponseWriter, r *http.Request) {
+	oldIntegration := context.Get(r, "integration").(db.Integration)
+	var integration db.Integration
+
+	if !helpers.Bind(w, r, &integration) {
+		return
+	}
+
+	if integration.ID != oldIntegration.ID {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Integration ID in body and URL must be the same",
+		})
+		return
+	}
+
+	if integration.ProjectID != oldIntegration.ProjectID {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Project ID in body and URL must be the same",
+		})
+		return
+	}
+
+	err := helpers.Store(r).UpdateIntegration(integration)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func DeleteIntegration(w http.ResponseWriter, r *http.Request) {
+	integration_id, err := helpers.GetIntParam("integration_id", w, r)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	project := context.Get(r, "project").(db.Project)
+
+	err = helpers.Store(r).DeleteIntegration(project.ID, integration_id)
+	if err == db.ErrInvalidOperation {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
+			"error": "Integration failed to be deleted",
+		})
+		return
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}

api/projects/integrationextractvalue.go

@@ -0,0 +1,178 @@
+package projects
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/ansible-semaphore/semaphore/api/helpers"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/context"
+	log "github.com/sirupsen/logrus"
+)
+
+func GetIntegrationExtractValue(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	valueId, err := helpers.GetIntParam("value_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid IntegrationExtractValue ID",
+		})
+		return
+	}
+
+	integration := context.Get(r, "integration").(db.Integration)
+	var value db.IntegrationExtractValue
+	value, err = helpers.Store(r).GetIntegrationExtractValue(project.ID, valueId, integration.ID)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": fmt.Sprintf("Failed to get IntegrationExtractValue, %v", err),
+		})
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, value)
+}
+
+func GetIntegrationExtractValues(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	integration := context.Get(r, "integration").(db.Integration)
+	values, err := helpers.Store(r).GetIntegrationExtractValues(project.ID, helpers.QueryParams(r.URL), integration.ID)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, values)
+}
+
+func AddIntegrationExtractValue(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	integration := context.Get(r, "integration").(db.Integration)
+
+	var value db.IntegrationExtractValue
+
+	if !helpers.Bind(w, r, &value) {
+		return
+	}
+
+	if value.IntegrationID != integration.ID {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Extractor ID in body and URL must be the same",
+		})
+		return
+	}
+
+	if err := value.Validate(); err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": err.Error(),
+		})
+		return
+	}
+
+	newValue, err := helpers.Store(r).CreateIntegrationExtractValue(project.ID, value)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusCreated, newValue)
+}
+
+func UpdateIntegrationExtractValue(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	valueId, err := helpers.GetIntParam("value_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Value ID",
+		})
+		return
+	}
+	integration := context.Get(r, "integration").(db.Integration)
+
+	var value db.IntegrationExtractValue
+	value, err = helpers.Store(r).GetIntegrationExtractValue(project.ID, valueId, integration.ID)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	if !helpers.Bind(w, r, &value) {
+		return
+	}
+
+	if value.ID != valueId {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Value ID in body and URL must be the same",
+		})
+		return
+	}
+
+	err = helpers.Store(r).UpdateIntegrationExtractValue(project.ID, value)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func GetIntegrationExtractValueRefs(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	valueId, err := helpers.GetIntParam("value_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Value ID",
+		})
+		return
+	}
+	integration := context.Get(r, "integration").(db.Integration)
+	var value db.IntegrationExtractValue
+	value, err = helpers.Store(r).GetIntegrationExtractValue(project.ID, valueId, integration.ID)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	refs, err := helpers.Store(r).GetIntegrationExtractValueRefs(project.ID, value.ID, value.IntegrationID)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, refs)
+}
+
+func DeleteIntegrationExtractValue(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	valueId, err := helpers.GetIntParam("value_id", w, r)
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Value ID",
+		})
+		return
+	}
+	integration := context.Get(r, "integration").(db.Integration)
+
+	if err != nil {
+		log.Error(err)
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
+			"error": "Integration Extract Value failed to be deleted",
+		})
+		return
+	}
+
+	err = helpers.Store(r).DeleteIntegrationExtractValue(project.ID, valueId, integration.ID)
+	if err == db.ErrInvalidOperation {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
+			"error": "Integration Extract Value failed to be deleted",
+		})
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}

api/projects/integrationmatcher.go

@@ -0,0 +1,170 @@
+package projects
+
+import (
+	//	"strconv"
+	"fmt"
+	"net/http"
+
+	"github.com/ansible-semaphore/semaphore/api/helpers"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/context"
+	log "github.com/sirupsen/logrus"
+)
+
+func GetIntegrationMatcher(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	matcher_id, err := helpers.GetIntParam("matcher_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Matcher ID",
+		})
+		return
+	}
+
+	integration := context.Get(r, "integration").(db.Integration)
+	var matcher db.IntegrationMatcher
+	matcher, err = helpers.Store(r).GetIntegrationMatcher(project.ID, matcher_id, integration.ID)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, matcher)
+}
+
+func GetIntegrationMatcherRefs(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	matcherId, err := helpers.GetIntParam("matcher_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Matcher ID",
+		})
+		return
+	}
+	integration := context.Get(r, "integration").(db.Integration)
+	var matcher db.IntegrationMatcher
+	matcher, err = helpers.Store(r).GetIntegrationMatcher(project.ID, matcherId, integration.ID)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	refs, err := helpers.Store(r).GetIntegrationMatcherRefs(project.ID, matcher.ID, matcher.IntegrationID)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, refs)
+}
+
+func GetIntegrationMatchers(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	integration := context.Get(r, "integration").(db.Integration)
+
+	matchers, err := helpers.Store(r).GetIntegrationMatchers(project.ID, helpers.QueryParams(r.URL), integration.ID)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, matchers)
+}
+
+func AddIntegrationMatcher(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	integration := context.Get(r, "integration").(db.Integration)
+
+	var matcher db.IntegrationMatcher
+	if !helpers.Bind(w, r, &matcher) {
+		return
+	}
+
+	if matcher.IntegrationID != integration.ID {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Extractor ID in body and URL must be the same",
+		})
+		return
+	}
+
+	err := matcher.Validate()
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": err.Error(),
+		})
+		return
+	}
+
+	newMatcher, err := helpers.Store(r).CreateIntegrationMatcher(project.ID, matcher)
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, newMatcher)
+}
+
+func UpdateIntegrationMatcher(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	matcherId, err := helpers.GetIntParam("matcher_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Matcher ID",
+		})
+		return
+	}
+	integration := context.Get(r, "integration").(db.Integration)
+
+	var matcher db.IntegrationMatcher
+
+	if !helpers.Bind(w, r, &matcher) {
+		return
+	}
+
+	log.Info(fmt.Sprintf("Updating API Matcher %v for Extractor %v, matcher ID: %v", matcherId, integration.ID, matcher.ID))
+
+	err = helpers.Store(r).UpdateIntegrationMatcher(project.ID, matcher)
+	log.Info(fmt.Sprintf("Err %s", err))
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func DeleteIntegrationMatcher(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	matcherId, err := helpers.GetIntParam("matcher_id", w, r)
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid Matcher ID",
+		})
+		return
+	}
+
+	integration := context.Get(r, "integration").(db.Integration)
+	var matcher db.IntegrationMatcher
+	matcher, err = helpers.Store(r).GetIntegrationMatcher(project.ID, matcherId, integration.ID)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	err = helpers.Store(r).DeleteIntegrationMatcher(project.ID, matcher.ID, integration.ID)
+	if err == db.ErrInvalidOperation {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
+			"error": "Integration Matcher failed to be deleted",
+		})
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}

api/projects/inventory.go

@@ -1,7 +1,7 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"net/http"

api/projects/keys.go

@@ -1,7 +1,7 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"net/http"

api/projects/project.go

@@ -3,6 +3,7 @@ package projects
 import (
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/mux"
 	"net/http"
 
 	"github.com/gorilla/context"
@@ -22,10 +23,10 @@ func ProjectMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
-		// check if user it project's team
+		// check if user in project's team
-		_, err = helpers.Store(r).GetProjectUser(projectID, user.ID)
+		projectUser, err := helpers.Store(r).GetProjectUser(projectID, user.ID)
 
-		if err != nil {
+		if !user.Admin && err != nil {
 			helpers.WriteError(w, err)
 			return
 		}
@@ -37,43 +38,44 @@ func ProjectMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
+		context.Set(r, "projectUserRole", projectUser.Role)
 		context.Set(r, "project", project)
 		next.ServeHTTP(w, r)
 	})
 }
 
-// MustBeAdmin ensures that the user has administrator rights
+// GetMustCanMiddleware ensures that the user has administrator rights
-func MustBeAdmin(next http.Handler) http.Handler {
+func GetMustCanMiddleware(permissions db.ProjectUserPermission) mux.MiddlewareFunc {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return func(next http.Handler) http.Handler {
-		project := context.Get(r, "project").(db.Project)
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		user := context.Get(r, "user").(*db.User)
+			me := context.Get(r, "user").(*db.User)
+			myRole := context.Get(r, "projectUserRole").(db.ProjectUserRole)
 
-		projectUser, err := helpers.Store(r).GetProjectUser(project.ID, user.ID)
+			if !me.Admin && r.Method != "GET" && r.Method != "HEAD" && !myRole.Can(permissions) {
+				w.WriteHeader(http.StatusForbidden)
+				return
+			}
 
-		if err == db.ErrNotFound {
+			next.ServeHTTP(w, r)
-			w.WriteHeader(http.StatusForbidden)
+		})
-			return
+	}
-		}
-
-		if err != nil {
-			helpers.WriteError(w, err)
-			return
-		}
-
-		if !projectUser.Admin {
-			w.WriteHeader(http.StatusForbidden)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	})
 }
 
-//GetProject returns a project details
+// GetProject returns a project details
 func GetProject(w http.ResponseWriter, r *http.Request) {
 	helpers.WriteJSON(w, http.StatusOK, context.Get(r, "project"))
 }
 
+func GetUserRole(w http.ResponseWriter, r *http.Request) {
+	var permissions struct {
+		Role        db.ProjectUserRole       `json:"role"`
+		Permissions db.ProjectUserPermission `json:"permissions"`
+	}
+	permissions.Role = context.Get(r, "projectUserRole").(db.ProjectUserRole)
+	permissions.Permissions = permissions.Role.GetPermissions()
+	helpers.WriteJSON(w, http.StatusOK, permissions)
+}
+
 // UpdateProject saves updated project details to the database
 func UpdateProject(w http.ResponseWriter, r *http.Request) {
 	project := context.Get(r, "project").(db.Project)

api/projects/projects.go

@@ -1,9 +1,10 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/ansible-semaphore/semaphore/util"
+	log "github.com/sirupsen/logrus"
 	"net/http"
 
 	"github.com/gorilla/context"
@@ -13,7 +14,13 @@ import (
 func GetProjects(w http.ResponseWriter, r *http.Request) {
 	user := context.Get(r, "user").(*db.User)
 
-	projects, err := helpers.Store(r).GetProjects(user.ID)
+	var err error
+	var projects []db.Project
+	if user.Admin {
+		projects, err = helpers.Store(r).GetAllProjects()
+	} else {
+		projects, err = helpers.Store(r).GetProjects(user.ID)
+	}
 
 	if err != nil {
 		helpers.WriteError(w, err)
@@ -23,37 +30,234 @@ func GetProjects(w http.ResponseWriter, r *http.Request) {
 	helpers.WriteJSON(w, http.StatusOK, projects)
 }
 
+func createDemoProject(projectID int, store db.Store) (err error) {
+	var noneKey db.AccessKey
+	var demoRepo db.Repository
+	var emptyEnv db.Environment
+
+	var buildInv db.Inventory
+	var devInv db.Inventory
+	var prodInv db.Inventory
+
+	noneKey, err = store.CreateAccessKey(db.AccessKey{
+		Name:      "None",
+		Type:      db.AccessKeyNone,
+		ProjectID: &projectID,
+	})
+
+	if err != nil {
+		return
+	}
+
+	vaultKey, err := store.CreateAccessKey(db.AccessKey{
+		Name:      "Vault Password",
+		Type:      db.AccessKeyLoginPassword,
+		ProjectID: &projectID,
+		LoginPassword: db.LoginPassword{
+			Password: "RAX6yKN7sBn2qDagRPls",
+		},
+	})
+
+	if err != nil {
+		return
+	}
+
+	demoRepo, err = store.CreateRepository(db.Repository{
+		Name:      "Demo",
+		ProjectID: projectID,
+		GitURL:    "https://github.com/semaphoreui/demo-project.git",
+		GitBranch: "main",
+		SSHKeyID:  noneKey.ID,
+	})
+
+	if err != nil {
+		return
+	}
+
+	emptyEnv, err = store.CreateEnvironment(db.Environment{
+		Name:      "Empty",
+		ProjectID: projectID,
+		JSON:      "{}",
+	})
+
+	if err != nil {
+		return
+	}
+
+	buildInv, err = store.CreateInventory(db.Inventory{
+		Name:      "Build",
+		ProjectID: projectID,
+		Inventory: "[builder]\nlocalhost ansible_connection=local",
+		Type:      "static",
+		SSHKeyID:  &noneKey.ID,
+	})
+
+	if err != nil {
+		return
+	}
+
+	devInv, err = store.CreateInventory(db.Inventory{
+		Name:      "Dev",
+		ProjectID: projectID,
+		Inventory: "invs/dev/hosts",
+		Type:      "file",
+		SSHKeyID:  &noneKey.ID,
+	})
+
+	if err != nil {
+		return
+	}
+
+	prodInv, err = store.CreateInventory(db.Inventory{
+		Name:      "Prod",
+		ProjectID: projectID,
+		Inventory: "invs/prod/hosts",
+		Type:      "file",
+		SSHKeyID:  &noneKey.ID,
+	})
+
+	var desc string
+
+	if err != nil {
+		return
+	}
+
+	desc = "This task pings the website to provide real word example of using Semaphore."
+	_, err = store.CreateTemplate(db.Template{
+		Name:          "Ping Site",
+		Playbook:      "ping.yml",
+		Description:   &desc,
+		ProjectID:     projectID,
+		InventoryID:   prodInv.ID,
+		EnvironmentID: &emptyEnv.ID,
+		RepositoryID:  demoRepo.ID,
+	})
+
+	if err != nil {
+		return
+	}
+
+	desc = "Creates artifact and store it in the cache."
+
+	var startVersion = "1.0.0"
+	buildTpl, err := store.CreateTemplate(db.Template{
+		Name:          "Build",
+		Playbook:      "build.yml",
+		Type:          db.TemplateBuild,
+		ProjectID:     projectID,
+		InventoryID:   buildInv.ID,
+		EnvironmentID: &emptyEnv.ID,
+		RepositoryID:  demoRepo.ID,
+		StartVersion:  &startVersion,
+	})
+
+	if err != nil {
+		return
+	}
+
+	_, err = store.CreateTemplate(db.Template{
+		Name:            "Deploy to Dev",
+		Type:            db.TemplateDeploy,
+		Playbook:        "deploy.yml",
+		ProjectID:       projectID,
+		InventoryID:     devInv.ID,
+		EnvironmentID:   &emptyEnv.ID,
+		RepositoryID:    demoRepo.ID,
+		BuildTemplateID: &buildTpl.ID,
+		Autorun:         true,
+		VaultKeyID:      &vaultKey.ID,
+	})
+
+	if err != nil {
+		return
+	}
+
+	_, err = store.CreateTemplate(db.Template{
+		Name:            "Deploy to Production",
+		Type:            db.TemplateDeploy,
+		Playbook:        "deploy.yml",
+		ProjectID:       projectID,
+		InventoryID:     prodInv.ID,
+		EnvironmentID:   &emptyEnv.ID,
+		RepositoryID:    demoRepo.ID,
+		BuildTemplateID: &buildTpl.ID,
+		VaultKeyID:      &vaultKey.ID,
+	})
+
+	return
+}
+
 // AddProject adds a new project to the database
 func AddProject(w http.ResponseWriter, r *http.Request) {
-	var body db.Project
 
 	user := context.Get(r, "user").(*db.User)
 
-	if !user.Admin {
+	if !user.Admin && !util.Config.NonAdminCanCreateProject {
 		log.Warn(user.Username + " is not permitted to edit users")
 		w.WriteHeader(http.StatusUnauthorized)
 		return
 	}
 
-	if !helpers.Bind(w, r, &body) {
+	var bodyWithDemo struct {
+		db.Project
+		Demo bool `json:"demo"`
+	}
+
+	if !helpers.Bind(w, r, &bodyWithDemo) {
 		return
 	}
 
-	body, err := helpers.Store(r).CreateProject(body)
+	body := bodyWithDemo.Project
+
+	store := helpers.Store(r)
+
+	body, err := store.CreateProject(body)
 	if err != nil {
 		helpers.WriteError(w, err)
 		return
 	}
 
-	_, err = helpers.Store(r).CreateProjectUser(db.ProjectUser{ProjectID: body.ID, UserID: user.ID, Admin: true})
+	_, err = store.CreateProjectUser(db.ProjectUser{ProjectID: body.ID, UserID: user.ID, Role: db.ProjectOwner})
 	if err != nil {
 		helpers.WriteError(w, err)
 		return
 	}
 
+	noneKey, err := store.CreateAccessKey(db.AccessKey{
+		Name:      "None",
+		Type:      db.AccessKeyNone,
+		ProjectID: &body.ID,
+	})
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	_, err = store.CreateInventory(db.Inventory{
+		Name:      "None",
+		ProjectID: body.ID,
+		Type:      "none",
+		SSHKeyID:  &noneKey.ID,
+	})
+
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	if bodyWithDemo.Demo {
+		err = createDemoProject(body.ID, store)
+
+		if err != nil {
+			helpers.WriteError(w, err)
+			return
+		}
+	}
+
 	desc := "Project Created"
 	oType := db.EventProject
-	_, err = helpers.Store(r).CreateEvent(db.Event{
+	_, err = store.CreateEvent(db.Event{
 		UserID:      &user.ID,
 		ProjectID:   &body.ID,
 		Description: &desc,

api/projects/repository.go

@@ -1,7 +1,7 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/ansible-semaphore/semaphore/util"

api/projects/schedules.go

@@ -1,7 +1,7 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/ansible-semaphore/semaphore/services/schedules"

api/projects/tasks.go

@@ -1,11 +1,11 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/ansible-semaphore/semaphore/util"
 	"github.com/gorilla/context"
+	log "github.com/sirupsen/logrus"
 	"net/http"
 	"strconv"
 )
@@ -121,6 +121,24 @@ func GetTaskOutput(w http.ResponseWriter, r *http.Request) {
 	helpers.WriteJSON(w, http.StatusOK, output)
 }
 
+func ConfirmTask(w http.ResponseWriter, r *http.Request) {
+	targetTask := context.Get(r, "task").(db.Task)
+	project := context.Get(r, "project").(db.Project)
+
+	if targetTask.ProjectID != project.ID {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	err := helpers.TaskPool(r).ConfirmTask(targetTask)
+	if err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
 func StopTask(w http.ResponseWriter, r *http.Request) {
 	targetTask := context.Get(r, "task").(db.Task)
 	project := context.Get(r, "project").(db.Project)
@@ -130,7 +148,15 @@ func StopTask(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err := helpers.TaskPool(r).StopTask(targetTask)
+	var stopObj struct {
+		Force bool `json:"force"`
+	}
+
+	if !helpers.Bind(w, r, &stopObj) {
+		return
+	}
+
+	err := helpers.TaskPool(r).StopTask(targetTask, stopObj.Force)
 	if err != nil {
 		helpers.WriteError(w, err)
 		return

api/projects/templates.go

@@ -1,7 +1,7 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/gorilla/context"

api/projects/users.go

@@ -1,13 +1,13 @@
 package projects
 
 import (
+	"fmt"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/context"
 	"net/http"
 	"strconv"
-
-	log "github.com/Sirupsen/logrus"
-	"github.com/gorilla/context"
 )
 
 // UserMiddleware ensures a user exists and loads it to the context
@@ -38,6 +38,13 @@ func UserMiddleware(next http.Handler) http.Handler {
 	})
 }
 
+type projUser struct {
+	ID       int                `json:"id"`
+	Username string             `json:"username"`
+	Name     string             `json:"name"`
+	Role     db.ProjectUserRole `json:"role"`
+}
+
 // GetUsers returns all users in a project
 func GetUsers(w http.ResponseWriter, r *http.Request) {
 
@@ -55,22 +62,42 @@ func GetUsers(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	helpers.WriteJSON(w, http.StatusOK, users)
+	var result = make([]projUser, 0)
+
+	for _, user := range users {
+		result = append(result, projUser{
+			ID:       user.ID,
+			Name:     user.Name,
+			Username: user.Username,
+			Role:     user.Role,
+		})
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, result)
 }
 
 // AddUser adds a user to a projects team in the database
 func AddUser(w http.ResponseWriter, r *http.Request) {
 	project := context.Get(r, "project").(db.Project)
 	var projectUser struct {
-		UserID int  `json:"user_id" binding:"required"`
+		UserID int                `json:"user_id" binding:"required"`
-		Admin  bool `json:"admin"`
+		Role   db.ProjectUserRole `json:"role"`
 	}
 
 	if !helpers.Bind(w, r, &projectUser) {
 		return
 	}
 
-	_, err := helpers.Store(r).CreateProjectUser(db.ProjectUser{ProjectID: project.ID, UserID: projectUser.UserID, Admin: projectUser.Admin})
+	if !projectUser.Role.IsValid() {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	_, err := helpers.Store(r).CreateProjectUser(db.ProjectUser{
+		ProjectID: project.ID,
+		UserID:    projectUser.UserID,
+		Role:      projectUser.Role,
+	})
 
 	if err != nil {
 		w.WriteHeader(http.StatusConflict)
@@ -82,7 +109,7 @@ func AddUser(w http.ResponseWriter, r *http.Request) {
 	desc := "User ID " + strconv.Itoa(projectUser.UserID) + " added to team"
 
 	_, err = helpers.Store(r).CreateEvent(db.Event{
-		UserID:		 &user.ID,
+		UserID:      &user.ID,
 		ProjectID:   &project.ID,
 		ObjectType:  &objType,
 		ObjectID:    &projectUser.UserID,
@@ -96,27 +123,32 @@ func AddUser(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
-// RemoveUser removes a user from a project team
+// removeUser removes a user from a project team
-func RemoveUser(w http.ResponseWriter, r *http.Request) {
+func removeUser(targetUser db.User, w http.ResponseWriter, r *http.Request) {
 	project := context.Get(r, "project").(db.Project)
-	projectUser := context.Get(r, "projectUser").(db.User)
+	me := context.Get(r, "user").(*db.User) // logged in user
+	myRole := context.Get(r, "projectUserRole").(db.ProjectUserRole)
 
-	err := helpers.Store(r).DeleteProjectUser(project.ID, projectUser.ID)
+	if !me.Admin && targetUser.ID == me.ID && myRole == db.ProjectOwner {
+		helpers.WriteError(w, fmt.Errorf("owner can not left the project"))
+		return
+	}
+
+	err := helpers.Store(r).DeleteProjectUser(project.ID, targetUser.ID)
 
 	if err != nil {
 		helpers.WriteError(w, err)
 		return
 	}
 
-	user := context.Get(r, "user").(*db.User)
 	objType := db.EventUser
-	desc := "User ID " + strconv.Itoa(projectUser.ID) + " removed from team"
+	desc := "User ID " + strconv.Itoa(targetUser.ID) + " removed from team"
 
 	_, err = helpers.Store(r).CreateEvent(db.Event{
-		UserID:      &user.ID,
+		UserID:      &me.ID,
 		ProjectID:   &project.ID,
 		ObjectType:  &objType,
-		ObjectID:    &projectUser.ID,
+		ObjectID:    &targetUser.ID,
 		Description: &desc,
 	})
 
@@ -127,18 +159,47 @@ func RemoveUser(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
-// MakeUserAdmin writes the admin flag to the users account
+// LeftProject removes a user from a project team
-func MakeUserAdmin(w http.ResponseWriter, r *http.Request) {
+func LeftProject(w http.ResponseWriter, r *http.Request) {
-	project := context.Get(r, "project").(db.Project)
+	me := context.Get(r, "user").(*db.User) // logged in user
-	user := context.Get(r, "projectUser").(db.User)
+	removeUser(*me, w, r)
-	admin := true
+}
 
-	if r.Method == "DELETE" {
+// RemoveUser removes a user from a project team
-		// strip admin
+func RemoveUser(w http.ResponseWriter, r *http.Request) {
-		admin = false
+	targetUser := context.Get(r, "projectUser").(db.User) // target user
+	removeUser(targetUser, w, r)
+}
+
+func UpdateUser(w http.ResponseWriter, r *http.Request) {
+	project := context.Get(r, "project").(db.Project)
+	me := context.Get(r, "user").(*db.User) // logged in user
+	targetUser := context.Get(r, "projectUser").(db.User)
+	targetUserRole := context.Get(r, "projectUserRole").(db.ProjectUserRole)
+
+	if !me.Admin && targetUser.ID == me.ID && targetUserRole == db.ProjectOwner {
+		helpers.WriteError(w, fmt.Errorf("owner can not change his role in the project"))
+		return
 	}
 
-	err := helpers.Store(r).UpdateProjectUser(db.ProjectUser{UserID: user.ID, ProjectID: project.ID, Admin: admin})
+	var projectUser struct {
+		Role db.ProjectUserRole `json:"role"`
+	}
+
+	if !helpers.Bind(w, r, &projectUser) {
+		return
+	}
+
+	if !projectUser.Role.IsValid() {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	err := helpers.Store(r).UpdateProjectUser(db.ProjectUser{
+		UserID:    targetUser.ID,
+		ProjectID: project.ID,
+		Role:      projectUser.Role,
+	})
 
 	if err != nil {
 		helpers.WriteError(w, err)

api/projects/views.go

@@ -1,7 +1,7 @@
 package projects
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"net/http"

api/router.go

@@ -1,28 +1,37 @@
 package api
 
 import (
+	"bytes"
+	"embed"
 	"fmt"
-	"github.com/ansible-semaphore/semaphore/api/helpers"
-	"github.com/ansible-semaphore/semaphore/db"
 	"net/http"
 	"os"
+	"path"
 	"strings"
+	"time"
 
+	"github.com/ansible-semaphore/semaphore/api/runners"
+
+	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/api/projects"
 	"github.com/ansible-semaphore/semaphore/api/sockets"
+	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/ansible-semaphore/semaphore/util"
-	"github.com/gobuffalo/packr"
 	"github.com/gorilla/mux"
 )
 
-var publicAssets2 = packr.NewBox("../web/dist")
+var startTime = time.Now().UTC()
 
+//go:embed public/*
+var publicAssets embed.FS
+
+// StoreMiddleware WTF?
 func StoreMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		store := helpers.Store(r)
-		var url = r.URL.String()
+		//var url = r.URL.String()
 
-		db.StoreSession(store, url, func() {
+		db.StoreSession(store, util.RandString(12), func() {
 			next.ServeHTTP(w, r)
 		})
 	})
@@ -49,15 +58,6 @@ func pongHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("pong"))
 }
 
-func notFoundHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Access-Control-Allow-Origin", r.Header.Get("Origin"))
-	w.Header().Set("Access-Control-Allow-Credentials", "true")
-	w.WriteHeader(http.StatusNotFound)
-	//nolint: errcheck
-	w.Write([]byte("404 not found"))
-	fmt.Println(r.Method, ":", r.URL.String(), "--> 404 Not Found")
-}
-
 // Route declares all routes
 func Route() *mux.Router {
 	r := mux.NewRouter()
@@ -78,11 +78,23 @@ func Route() *mux.Router {
 	pingRouter.Methods("GET", "HEAD").HandlerFunc(pongHandler)
 
 	publicAPIRouter := r.PathPrefix(webPath + "api").Subrouter()
-
 	publicAPIRouter.Use(StoreMiddleware, JSONMiddleware)
 
-	publicAPIRouter.HandleFunc("/auth/login", login).Methods("POST")
+	publicAPIRouter.HandleFunc("/runners", runners.RegisterRunner).Methods("POST")
+	publicAPIRouter.HandleFunc("/auth/login", login).Methods("GET", "POST")
 	publicAPIRouter.HandleFunc("/auth/logout", logout).Methods("POST")
+	publicAPIRouter.HandleFunc("/auth/oidc/{provider}/login", oidcLogin).Methods("GET")
+	publicAPIRouter.HandleFunc("/auth/oidc/{provider}/redirect", oidcRedirect).Methods("GET")
+	publicAPIRouter.HandleFunc("/auth/oidc/{provider}/redirect/{redirect_path:.*}", oidcRedirect).Methods("GET")
+
+	routersAPI := r.PathPrefix(webPath + "api").Subrouter()
+	routersAPI.Use(StoreMiddleware, JSONMiddleware, runners.RunnerMiddleware)
+	routersAPI.Path("/runners/{runner_id}").HandlerFunc(runners.GetRunner).Methods("GET", "HEAD")
+	routersAPI.Path("/runners/{runner_id}").HandlerFunc(runners.UpdateRunner).Methods("PUT")
+
+	publicWebHookRouter := r.PathPrefix(webPath + "api/project/{project_id}/integrations/{integration_id}").Subrouter()
+	publicWebHookRouter.Use(StoreMiddleware, JSONMiddleware, projects.IntegrationMiddleware)
+	publicWebHookRouter.HandleFunc("/endpoint", ReceiveIntegration).Methods("POST", "GET", "OPTIONS")
 
 	authenticatedWS := r.PathPrefix(webPath + "api").Subrouter()
 	authenticatedWS.Use(JSONMiddleware, authenticationWithStore)
@@ -96,6 +108,7 @@ func Route() *mux.Router {
 
 	authenticatedAPI.Path("/projects").HandlerFunc(projects.GetProjects).Methods("GET", "HEAD")
 	authenticatedAPI.Path("/projects").HandlerFunc(projects.AddProject).Methods("POST")
+	authenticatedAPI.Path("/projects/restore").HandlerFunc(projects.Restore).Methods("POST")
 	authenticatedAPI.Path("/events").HandlerFunc(getAllEvents).Methods("GET", "HEAD")
 	authenticatedAPI.HandleFunc("/events/last", getLastEvents).Methods("GET", "HEAD")
 
@@ -123,8 +136,23 @@ func Route() *mux.Router {
 	projectGet.Use(projects.ProjectMiddleware)
 	projectGet.Methods("GET", "HEAD").HandlerFunc(projects.GetProject)
 
+	//
+	// Start and Stop tasks
+	projectTaskStart := authenticatedAPI.PathPrefix("/project/{project_id}").Subrouter()
+	projectTaskStart.Use(projects.ProjectMiddleware, projects.GetMustCanMiddleware(db.CanRunProjectTasks))
+	projectTaskStart.Path("/tasks").HandlerFunc(projects.AddTask).Methods("POST")
+
+	projectTaskStop := authenticatedAPI.PathPrefix("/project/{project_id}").Subrouter()
+	projectTaskStop.Use(projects.ProjectMiddleware, projects.GetTaskMiddleware, projects.GetMustCanMiddleware(db.CanRunProjectTasks))
+	projectTaskStop.HandleFunc("/tasks/{task_id}/stop", projects.StopTask).Methods("POST")
+	projectTaskStop.HandleFunc("/tasks/{task_id}/confirm", projects.ConfirmTask).Methods("POST")
+
+	//
+	// Project resources CRUD
 	projectUserAPI := authenticatedAPI.PathPrefix("/project/{project_id}").Subrouter()
-	projectUserAPI.Use(projects.ProjectMiddleware)
+	projectUserAPI.Use(projects.ProjectMiddleware, projects.GetMustCanMiddleware(db.CanManageProjectResources))
+
+	projectUserAPI.Path("/role").HandlerFunc(projects.GetUserRole).Methods("GET", "HEAD")
 
 	projectUserAPI.Path("/events").HandlerFunc(getAllEvents).Methods("GET", "HEAD")
 	projectUserAPI.HandleFunc("/events/last", getLastEvents).Methods("GET", "HEAD")
@@ -145,7 +173,6 @@ func Route() *mux.Router {
 
 	projectUserAPI.Path("/tasks").HandlerFunc(projects.GetAllTasks).Methods("GET", "HEAD")
 	projectUserAPI.HandleFunc("/tasks/last", projects.GetLastTasks).Methods("GET", "HEAD")
-	projectUserAPI.Path("/tasks").HandlerFunc(projects.AddTask).Methods("POST")
 
 	projectUserAPI.Path("/templates").HandlerFunc(projects.GetTemplates).Methods("GET", "HEAD")
 	projectUserAPI.Path("/templates").HandlerFunc(projects.AddTemplate).Methods("POST")
@@ -157,23 +184,37 @@ func Route() *mux.Router {
 	projectUserAPI.Path("/views").HandlerFunc(projects.AddView).Methods("POST")
 	projectUserAPI.Path("/views/positions").HandlerFunc(projects.SetViewPositions).Methods("POST")
 
+	projectUserAPI.Path("/integrations").HandlerFunc(projects.GetIntegrations).Methods("GET", "HEAD")
+	projectUserAPI.Path("/integrations").HandlerFunc(projects.AddIntegration).Methods("POST")
+	projectUserAPI.Path("/backup").HandlerFunc(projects.GetBackup).Methods("GET", "HEAD")
+
+	//
+	// Updating and deleting project
 	projectAdminAPI := authenticatedAPI.Path("/project/{project_id}").Subrouter()
-	projectAdminAPI.Use(projects.ProjectMiddleware, projects.MustBeAdmin)
+	projectAdminAPI.Use(projects.ProjectMiddleware, projects.GetMustCanMiddleware(db.CanUpdateProject))
 	projectAdminAPI.Methods("PUT").HandlerFunc(projects.UpdateProject)
 	projectAdminAPI.Methods("DELETE").HandlerFunc(projects.DeleteProject)
 
+	meAPI := authenticatedAPI.Path("/project/{project_id}/me").Subrouter()
+	meAPI.Use(projects.ProjectMiddleware)
+	meAPI.HandleFunc("", projects.LeftProject).Methods("DELETE")
+
+	//
+	// Manage project users
 	projectAdminUsersAPI := authenticatedAPI.PathPrefix("/project/{project_id}").Subrouter()
-	projectAdminUsersAPI.Use(projects.ProjectMiddleware, projects.MustBeAdmin)
+
+	projectAdminUsersAPI.Use(projects.ProjectMiddleware, projects.GetMustCanMiddleware(db.CanManageProjectUsers))
 	projectAdminUsersAPI.Path("/users").HandlerFunc(projects.AddUser).Methods("POST")
 
 	projectUserManagement := projectAdminUsersAPI.PathPrefix("/users").Subrouter()
 	projectUserManagement.Use(projects.UserMiddleware)
 
 	projectUserManagement.HandleFunc("/{user_id}", projects.GetUsers).Methods("GET", "HEAD")
-	projectUserManagement.HandleFunc("/{user_id}/admin", projects.MakeUserAdmin).Methods("POST")
+	projectUserManagement.HandleFunc("/{user_id}", projects.UpdateUser).Methods("PUT")
-	projectUserManagement.HandleFunc("/{user_id}/admin", projects.MakeUserAdmin).Methods("DELETE")
 	projectUserManagement.HandleFunc("/{user_id}", projects.RemoveUser).Methods("DELETE")
 
+	//
+	// Project resources CRUD (continue)
 	projectKeyManagement := projectUserAPI.PathPrefix("/keys").Subrouter()
 	projectKeyManagement.Use(projects.KeyMiddleware)
 
@@ -223,7 +264,6 @@ func Route() *mux.Router {
 	projectTaskManagement.HandleFunc("/{task_id}/output", projects.GetTaskOutput).Methods("GET", "HEAD")
 	projectTaskManagement.HandleFunc("/{task_id}", projects.GetTask).Methods("GET", "HEAD")
 	projectTaskManagement.HandleFunc("/{task_id}", projects.RemoveTask).Methods("DELETE")
-	projectTaskManagement.HandleFunc("/{task_id}/stop", projects.StopTask).Methods("POST")
 
 	projectScheduleManagement := projectUserAPI.PathPrefix("/schedules").Subrouter()
 	projectScheduleManagement.Use(projects.SchedulesMiddleware)
@@ -238,6 +278,29 @@ func Route() *mux.Router {
 	projectViewManagement.HandleFunc("/{view_id}", projects.RemoveView).Methods("DELETE")
 	projectViewManagement.HandleFunc("/{view_id}/templates", projects.GetViewTemplates).Methods("GET", "HEAD")
 
+	projectIntegrationsAPI := projectUserAPI.PathPrefix("/integrations").Subrouter()
+
+	projectIntegrationsAPI.Use(projects.ProjectMiddleware, projects.IntegrationMiddleware)
+	projectIntegrationsAPI.HandleFunc("/{integration_id}", projects.UpdateIntegration).Methods("PUT")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}", projects.DeleteIntegration).Methods("DELETE")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}", projects.GetIntegration).Methods("GET")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/refs", projects.GetIntegrationRefs).Methods("GET", "HEAD")
+
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/matchers", projects.GetIntegrationMatchers).Methods("GET", "HEAD")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/matchers", projects.AddIntegrationMatcher).Methods("POST")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/values", projects.GetIntegrationExtractValues).Methods("GET", "HEAD")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/values", projects.AddIntegrationExtractValue).Methods("POST")
+
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/matchers/{matcher_id}", projects.GetIntegrationMatcher).Methods("GET", "HEAD")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/matchers/{matcher_id}", projects.UpdateIntegrationMatcher).Methods("PUT")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/matchers/{matcher_id}", projects.DeleteIntegrationMatcher).Methods("DELETE")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/matchers/{matcher_id}/refs", projects.GetIntegrationMatcherRefs).Methods("GET", "HEAD")
+
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/values/{value_id}", projects.GetIntegrationExtractValue).Methods("GET", "HEAD")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/values/{value_id}", projects.UpdateIntegrationExtractValue).Methods("PUT")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/values/{value_id}", projects.DeleteIntegrationExtractValue).Methods("DELETE")
+	projectIntegrationsAPI.HandleFunc("/{integration_id}/values/{value_id}/refs", projects.GetIntegrationExtractValueRefs).Methods("GET")
+
 	if os.Getenv("DEBUG") == "1" {
 		defer debugPrintRoutes(r)
 	}
@@ -276,82 +339,92 @@ func debugPrintRoutes(r *mux.Router) {
 	}
 }
 
-// nolint: gocyclo
 func servePublic(w http.ResponseWriter, r *http.Request) {
 	webPath := "/"
 	if util.WebHostURL != nil {
-		webPath = util.WebHostURL.RequestURI()
+		webPath = util.WebHostURL.Path
+		if !strings.HasSuffix(webPath, "/") {
+			webPath += "/"
+		}
 	}
 
-	path := r.URL.Path
+	reqPath := r.URL.Path
+	apiPath := path.Join(webPath, "api")
 
-	if path == webPath+"api" || strings.HasPrefix(path, webPath+"api/") {
+	if reqPath == apiPath || strings.HasPrefix(reqPath, apiPath) {
-		w.WriteHeader(http.StatusNotFound)
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
 		return
 	}
 
-	if !strings.Contains(path, ".") {
+	if !strings.Contains(reqPath, ".") {
-		path = "/index.html"
+		serveFile(w, r, "index.html")
+		return
 	}
 
-	path = strings.Replace(path, webPath+"/", "", 1)
+	newPath := strings.Replace(
-	split := strings.Split(path, ".")
+		reqPath,
-	suffix := split[len(split)-1]
+		webPath,
+		"",
+		1,
+	)
 
-	var res []byte
+	serveFile(w, r, newPath)
-	var err error
+}
 
-	res, err = publicAssets2.MustBytes(path)
+func serveFile(w http.ResponseWriter, r *http.Request, name string) {
+	res, err := publicAssets.ReadFile(
+		fmt.Sprintf("public/%s", name),
+	)
 
 	if err != nil {
-		notFoundHandler(w, r)
+		http.Error(
+			w,
+			http.StatusText(http.StatusNotFound),
+			http.StatusNotFound,
+		)
+
 		return
 	}
 
-	// replace base path
+	if util.WebHostURL != nil && name == "index.html" {
-	if util.WebHostURL != nil && path == "/index.html" {
 		baseURL := util.WebHostURL.String()
+
 		if !strings.HasSuffix(baseURL, "/") {
 			baseURL += "/"
 		}
-		res = []byte(strings.Replace(string(res),
+
-			"<base href=\"/\">",
+		res = []byte(
-			"<base href=\""+baseURL+"\">",
+			strings.Replace(
-			1))
+				string(res),
+				`<base href="/">`,
+				fmt.Sprintf(`<base href="%s">`, baseURL),
+				1,
+			),
+		)
 	}
 
-	contentType := "text/plain"
+	if !strings.HasSuffix(name, ".html") {
-	switch suffix {
+		w.Header().Add(
-	case "png":
+			"Cache-Control",
-		contentType = "image/png"
+			fmt.Sprintf("max-age=%d, public, must-revalidate, proxy-revalidate", 24*time.Hour),
-	case "jpg", "jpeg":
+		)
-		contentType = "image/jpeg"
-	case "gif":
-		contentType = "image/gif"
-	case "js":
-		contentType = "application/javascript"
-	case "css":
-		contentType = "text/css"
-	case "woff":
-		contentType = "application/x-font-woff"
-	case "ttf":
-		contentType = "application/x-font-ttf"
-	case "otf":
-		contentType = "application/x-font-otf"
-	case "html":
-		contentType = "text/html"
 	}
 
-	w.Header().Set("content-type", contentType)
+	http.ServeContent(
-	_, err = w.Write(res)
+		w,
-	util.LogWarning(err)
+		r,
+		name,
+		startTime,
+		bytes.NewReader(
+			res,
+		),
+	)
 }
 
 func getSystemInfo(w http.ResponseWriter, r *http.Request) {
 	body := map[string]interface{}{
 		"version": util.Version,
 		"ansible": util.AnsibleVersion(),
-		"demo":    util.Config.DemoMode,
 	}
 
 	helpers.WriteJSON(w, http.StatusOK, body)

api/runners/runners.go

@@ -0,0 +1,198 @@
+package runners
+
+import (
+	"github.com/ansible-semaphore/semaphore/api/helpers"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/ansible-semaphore/semaphore/lib"
+	"github.com/ansible-semaphore/semaphore/services/runners"
+	"github.com/ansible-semaphore/semaphore/util"
+	"github.com/gorilla/context"
+	"net/http"
+)
+
+func RunnerMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		token := r.Header.Get("X-API-Token")
+
+		if token == "" {
+			helpers.WriteJSON(w, http.StatusUnauthorized, map[string]string{
+				"error": "Invalid token",
+			})
+			return
+		}
+
+		runnerID, err := helpers.GetIntParam("runner_id", w, r)
+
+		if err != nil {
+			helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+				"error": "runner_id required",
+			})
+			return
+		}
+
+		store := helpers.Store(r)
+
+		runner, err := store.GetGlobalRunner(runnerID)
+
+		if err != nil {
+			helpers.WriteJSON(w, http.StatusNotFound, map[string]string{
+				"error": "Runner not found",
+			})
+			return
+		}
+
+		if runner.Token != token {
+			helpers.WriteJSON(w, http.StatusUnauthorized, map[string]string{
+				"error": "Invalid token",
+			})
+			return
+		}
+
+		context.Set(r, "runner", runner)
+		next.ServeHTTP(w, r)
+	})
+}
+
+func GetRunner(w http.ResponseWriter, r *http.Request) {
+	runner := context.Get(r, "runner").(db.Runner)
+
+	data := runners.RunnerState{
+		AccessKeys: make(map[int]db.AccessKey),
+	}
+
+	tasks := helpers.TaskPool(r).GetRunningTasks()
+
+	for _, tsk := range tasks {
+		if tsk.RunnerID != runner.ID {
+			continue
+		}
+
+		if tsk.Task.Status == lib.TaskStartingStatus {
+
+			data.NewJobs = append(data.NewJobs, runners.JobData{
+				Username:        tsk.Username,
+				IncomingVersion: tsk.IncomingVersion,
+				Task:            tsk.Task,
+				Template:        tsk.Template,
+				Inventory:       tsk.Inventory,
+				Repository:      tsk.Repository,
+				Environment:     tsk.Environment,
+			})
+
+			if tsk.Inventory.SSHKeyID != nil {
+				err := tsk.Inventory.SSHKey.DeserializeSecret()
+				if err != nil {
+					// TODO: return error
+				}
+				data.AccessKeys[*tsk.Inventory.SSHKeyID] = tsk.Inventory.SSHKey
+			}
+
+			if tsk.Inventory.BecomeKeyID != nil {
+				err := tsk.Inventory.BecomeKey.DeserializeSecret()
+				if err != nil {
+					// TODO: return error
+				}
+				data.AccessKeys[*tsk.Inventory.BecomeKeyID] = tsk.Inventory.BecomeKey
+			}
+
+			if tsk.Template.VaultKeyID != nil {
+				err := tsk.Template.VaultKey.DeserializeSecret()
+				if err != nil {
+					// TODO: return error
+				}
+				data.AccessKeys[*tsk.Template.VaultKeyID] = tsk.Template.VaultKey
+			}
+
+			data.AccessKeys[tsk.Repository.SSHKeyID] = tsk.Repository.SSHKey
+
+		} else {
+			data.CurrentJobs = append(data.CurrentJobs, runners.JobState{
+				ID:     tsk.Task.ID,
+				Status: tsk.Task.Status,
+			})
+		}
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, data)
+}
+
+func UpdateRunner(w http.ResponseWriter, r *http.Request) {
+
+	runner := context.Get(r, "runner").(db.Runner)
+
+	var body runners.RunnerProgress
+
+	if !helpers.Bind(w, r, &body) {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid format",
+		})
+		return
+	}
+
+	taskPool := helpers.TaskPool(r)
+
+	if body.Jobs == nil {
+		w.WriteHeader(http.StatusNoContent)
+		return
+	}
+
+	for _, job := range body.Jobs {
+		tsk := taskPool.GetTask(job.ID)
+
+		if tsk == nil {
+			// TODO: log
+			continue
+		}
+
+		if tsk.RunnerID != runner.ID {
+			// TODO: add error message
+			continue
+		}
+
+		for _, logRecord := range job.LogRecords {
+			tsk.Log2(logRecord.Message, logRecord.Time)
+		}
+
+		tsk.SetStatus(job.Status)
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func RegisterRunner(w http.ResponseWriter, r *http.Request) {
+	var register runners.RunnerRegistration
+
+	if !helpers.Bind(w, r, &register) {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid format",
+		})
+		return
+	}
+
+	if util.Config.RunnerRegistrationToken == "" || register.RegistrationToken != util.Config.RunnerRegistrationToken {
+		helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
+			"error": "Invalid registration token",
+		})
+		return
+	}
+
+	runner, err := helpers.Store(r).CreateRunner(db.Runner{
+		Webhook:          register.Webhook,
+		MaxParallelTasks: register.MaxParallelTasks,
+	})
+
+	if err != nil {
+		helpers.WriteJSON(w, http.StatusInternalServerError, map[string]string{
+			"error": "Unexpected error",
+		})
+		return
+	}
+
+	res := runners.RunnerConfig{
+		RunnerID: runner.ID,
+		Token:    runner.Token,
+	}
+
+	helpers.WriteJSON(w, http.StatusOK, res)
+}

api/sockets/handler.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"time"
 
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/util"
 	"github.com/gorilla/context"
 	"github.com/gorilla/websocket"

api/user.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/ansible-semaphore/semaphore/util"
 	"github.com/gorilla/context"
 	"github.com/gorilla/mux"
 	"io"
@@ -18,7 +19,17 @@ func getUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	helpers.WriteJSON(w, http.StatusOK, context.Get(r, "user"))
+	var user struct {
+		db.User
+		CanCreateProject   bool `json:"can_create_project"`
+		IntegrationsEnable bool `json:"integrations_enable"`
+	}
+
+	user.User = *context.Get(r, "user").(*db.User)
+	user.CanCreateProject = user.Admin || util.Config.NonAdminCanCreateProject
+	user.IntegrationsEnable = util.Config.IntegrationsEnable
+
+	helpers.WriteJSON(w, http.StatusOK, user)
 }
 
 func getAPITokens(w http.ResponseWriter, r *http.Request) {

api/users.go

@@ -1,7 +1,7 @@
 package api
 
 import (
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api/helpers"
 	"github.com/ansible-semaphore/semaphore/db"
 	"net/http"
@@ -10,14 +10,35 @@ import (
 	"github.com/gorilla/context"
 )
 
+type minimalUser struct {
+	ID       int    `json:"id"`
+	Username string `json:"username"`
+	Name     string `json:"name"`
+}
+
 func getUsers(w http.ResponseWriter, r *http.Request) {
+	currentUser := context.Get(r, "user").(*db.User)
 	users, err := helpers.Store(r).GetUsers(db.RetrieveQueryParams{})
 
 	if err != nil {
 		panic(err)
 	}
 
-	helpers.WriteJSON(w, http.StatusOK, users)
+	if currentUser.Admin {
+		helpers.WriteJSON(w, http.StatusOK, users)
+	} else {
+		var result = make([]minimalUser, 0)
+
+		for _, user := range users {
+			result = append(result, minimalUser{
+				ID:       user.ID,
+				Name:     user.Name,
+				Username: user.Username,
+			})
+		}
+
+		helpers.WriteJSON(w, http.StatusOK, result)
+	}
 }
 
 func addUser(w http.ResponseWriter, r *http.Request) {
@@ -73,7 +94,7 @@ func getUserMiddleware(next http.Handler) http.Handler {
 }
 
 func updateUser(w http.ResponseWriter, r *http.Request) {
-	oldUser := context.Get(r, "_user").(db.User)
+	targetUser := context.Get(r, "_user").(db.User)
 	editor := context.Get(r, "user").(*db.User)
 
 	var user db.UserWithPwd
@@ -81,25 +102,25 @@ func updateUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if !editor.Admin && editor.ID != oldUser.ID {
+	if !editor.Admin && editor.ID != targetUser.ID {
 		log.Warn(editor.Username + " is not permitted to edit users")
 		w.WriteHeader(http.StatusUnauthorized)
 		return
 	}
 
-	if editor.ID == oldUser.ID && oldUser.Admin != user.Admin {
+	if editor.ID == targetUser.ID && targetUser.Admin != user.Admin {
 		log.Warn("User can't edit his own role")
 		w.WriteHeader(http.StatusUnauthorized)
 		return
 	}
 
-	if oldUser.External && oldUser.Username != user.Username {
+	if targetUser.External && targetUser.Username != user.Username {
-		log.Warn("Username is not editable for external LDAP users")
+		log.Warn("Username is not editable for external users")
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
 
-	user.ID = oldUser.ID
+	user.ID = targetUser.ID
 	if err := helpers.Store(r).UpdateUser(user); err != nil {
 		log.Error(err.Error())
 		w.WriteHeader(http.StatusBadRequest)
@@ -124,7 +145,7 @@ func updateUserPassword(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if user.External {
-		log.Warn("Password is not editable for external LDAP users")
+		log.Warn("Password is not editable for external users")
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}

cli/cmd/root.go

@@ -2,7 +2,7 @@ package cmd
 
 import (
 	"fmt"
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"github.com/ansible-semaphore/semaphore/api"
 	"github.com/ansible-semaphore/semaphore/api/sockets"
 	"github.com/ansible-semaphore/semaphore/db"
@@ -15,6 +15,7 @@ import (
 	"github.com/spf13/cobra"
 	"net/http"
 	"os"
+	"strings"
 )
 
 var configPath string
@@ -48,6 +49,12 @@ func runService() {
 
 	util.Config.PrintDbInfo()
 
+	port := util.Config.Port
+
+	if !strings.HasPrefix(port, ":") {
+		port = ":" + port
+	}
+
 	fmt.Printf("Tmp Path (projects home) %v\n", util.Config.TmpPath)
 	fmt.Printf("Semaphore %v\n", util.Version)
 	fmt.Printf("Interface %v\n", util.Config.Interface)
@@ -81,7 +88,7 @@ func runService() {
 		store.Close("root")
 	}
 
-	err := http.ListenAndServe(util.Config.Interface+util.Config.Port, cropTrailingSlashMiddleware(router))
+	err := http.ListenAndServe(util.Config.Interface+port, cropTrailingSlashMiddleware(router))
 
 	if err != nil {
 		log.Panic(err)
@@ -95,16 +102,6 @@ func createStore(token string) db.Store {
 
 	store.Connect(token)
 
-	//if err := store.Connect(token); err != nil {
-	//	switch err {
-	//	case bbolt.ErrTimeout:
-	//		fmt.Println("\n BoltDB supports only one connection at a time. You should stop Semaphore to use CLI.")
-	//	default:
-	//		fmt.Println("\n Have you run `semaphore setup`?")
-	//	}
-	//	os.Exit(1)
-	//}
-
 	err := db.Migrate(store)
 
 	if err != nil {

cli/cmd/runner.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"github.com/ansible-semaphore/semaphore/services/runners"
+	"github.com/ansible-semaphore/semaphore/util"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(runnerCmd)
+}
+
+func runRunner() {
+	util.ConfigInit(configPath)
+
+	taskPool := runners.JobPool{}
+
+	taskPool.Run()
+}
+
+var runnerCmd = &cobra.Command{
+	Use:   "runner",
+	Short: "Run in runner mode",
+	Run: func(cmd *cobra.Command, args []string) {
+		runRunner()
+	},
+}

cli/cmd/vault.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"os"
+)
+
+type vaultArgs struct {
+	oldKey string
+}
+
+var targetVaultArgs vaultArgs
+
+func init() {
+	rootCmd.AddCommand(vaultCmd)
+}
+
+var vaultCmd = &cobra.Command{
+	Use:   "vault",
+	Short: "Manage access keys and other secrets",
+	Run: func(cmd *cobra.Command, args []string) {
+		_ = cmd.Help()
+		os.Exit(0)
+	},
+}

cli/cmd/vault_rekey.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	vaultRekeyCmd.PersistentFlags().StringVar(&targetVaultArgs.oldKey, "old-key", "", "Old encryption key")
+
+	vaultCmd.AddCommand(vaultRekeyCmd)
+}
+
+var vaultRekeyCmd = &cobra.Command{
+	Use:   "rekey",
+	Short: "Re-encrypt Key Store in database with using current encryption key",
+	Long: "To update the encryption key, modify it within the configuration file and " +
+		"then employ the 'vault rekey --old-key <old-key>' command to ensure the re-encryption of the " +
+		"pre-existing keys stored in the database.",
+	Run: func(cmd *cobra.Command, args []string) {
+		store := createStore("")
+		defer store.Close("")
+
+		err := store.RekeyAccessKeys(targetVaultArgs.oldKey)
+
+		if err != nil {
+			panic(err)
+		}
+
+	},
+}

cli/setup/setup.go

@@ -2,12 +2,11 @@ package setup
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/ansible-semaphore/semaphore/util"
 )
@@ -50,7 +49,7 @@ func InteractiveSetup(conf *util.ConfigType) {
 	askValue("Playbook path", defaultPlaybookPath, &conf.TmpPath)
 	conf.TmpPath = filepath.Clean(conf.TmpPath)
 
-	askValue("Web root URL (optional, see https://github.com/ansible-semaphore/semaphore/wiki/Web-root-URL)", "", &conf.WebHost)
+	askValue("Public URL (optional, example: https://example.com/semaphore)", "", &conf.WebHost)
 
 	askConfirmation("Enable email alerts?", false, &conf.EmailAlert)
 	if conf.EmailAlert {
@@ -70,6 +69,11 @@ func InteractiveSetup(conf *util.ConfigType) {
 		askValue("Slack Webhook URL", "", &conf.SlackUrl)
 	}
 
+	askConfirmation("Enable Microsoft Team Channel alerts?", false, &conf.MicrosoftTeamsAlert)
+	if conf.MicrosoftTeamsAlert {
+		askValue("Microsoft Teams Webhook URL", "", &conf.MicrosoftTeamsUrl)
+	}
+
 	askConfirmation("Enable LDAP authentication?", false, &conf.LdapEnable)
 	if conf.LdapEnable {
 		askValue("LDAP server host", "localhost:389", &conf.LdapServer)
@@ -151,7 +155,7 @@ func SaveConfig(config *util.ConfigType) (configPath string) {
 	}
 
 	configPath = filepath.Join(configDirectory, "config.json")
-	if err = ioutil.WriteFile(configPath, bytes, 0644); err != nil {
+	if err = os.WriteFile(configPath, bytes, 0644); err != nil {
 		panic(err)
 	}
 

config-runner.json

@@ -0,0 +1,13 @@
+{
+ 	"bolt": {
+ 		"host": "/Users/fiftin/go/src/github.com/ansible-semaphore/semaphore/database.boltdb"
+ 	},
+
+ 	"dialect": "bolt",
+ 	"tmp_path": "/var/folders/x1/d7p_yr4j7g57_r2r8s0ll_1r0000gn/T/semaphore",
+
+	"runner": {
+		"config_file": "/tmp/semaphore-runner.json",
+		"api_url": "http://localhost:3000/api"
+	}
+}

db/AccessKey.go

@@ -7,11 +7,13 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"github.com/ansible-semaphore/semaphore/lib"
 	"io"
-	"io/ioutil"
 	"math/big"
 	"os"
+	"path"
 	"strconv"
+	"time"
 
 	"github.com/ansible-semaphore/semaphore/util"
 )
@@ -40,8 +42,6 @@ type AccessKey struct {
 	LoginPassword  LoginPassword `db:"-" json:"login_password"`
 	SshKey         SshKey        `db:"-" json:"ssh"`
 	OverrideSecret bool          `db:"-" json:"override_secret"`
-
-	InstallationKey int64 `db:"-" json:"-"`
 }
 
 type LoginPassword struct {
@@ -64,62 +64,104 @@ const (
 	AccessKeyRoleGit
 )
 
-func (key *AccessKey) Install(usage AccessKeyRole) error {
+type AccessKeyInstallation struct {
-	rnd, err := rand.Int(rand.Reader, big.NewInt(1000000000))
+	InstallationKey int64
-	if err != nil {
+	SshAgent        *lib.SshAgent
-		return err
+}
+
+func (key AccessKeyInstallation) Destroy() error {
+	if key.SshAgent != nil {
+		return key.SshAgent.Close()
 	}
 
-	key.InstallationKey = rnd.Int64()
+	installPath := key.GetPath()
-
+	_, err := os.Stat(installPath)
-	if key.Type == AccessKeyNone {
+	if os.IsNotExist(err) {
 		return nil
 	}
+	return os.Remove(installPath)
+}
 
-	path := key.GetPath()
+// GetPath returns the location of the access key once written to disk
+func (key AccessKeyInstallation) GetPath() string {
+	return util.Config.TmpPath + "/access_key_" + strconv.FormatInt(key.InstallationKey, 10)
+}
+
+func (key *AccessKey) startSshAgent(logger lib.Logger) (lib.SshAgent, error) {
+
+	sshAgent := lib.SshAgent{
+		Logger: logger,
+		Keys: []lib.SshAgentKey{
+			{
+				Key:        []byte(key.SshKey.PrivateKey),
+				Passphrase: []byte(key.SshKey.Passphrase),
+			},
+		},
+		SocketFile: path.Join(util.Config.TmpPath, fmt.Sprintf("ssh-agent-%d-%d.sock", key.ID, time.Now().Unix())),
+	}
+
+	return sshAgent, sshAgent.Listen()
+}
+
+func (key *AccessKey) Install(usage AccessKeyRole, logger lib.Logger) (installation AccessKeyInstallation, err error) {
+	rnd, err := rand.Int(rand.Reader, big.NewInt(1000000000))
+	if err != nil {
+		return
+	}
+
+	installation.InstallationKey = rnd.Int64()
+
+	if key.Type == AccessKeyNone {
+		return
+	}
+
+	installationPath := installation.GetPath()
 
 	err = key.DeserializeSecret()
 
 	if err != nil {
-		return err
+		return
 	}
 
 	switch usage {
 	case AccessKeyRoleGit:
 		switch key.Type {
 		case AccessKeySSH:
-			if key.SshKey.Passphrase != "" {
+			var agent lib.SshAgent
-				return fmt.Errorf("ssh key with passphrase not supported")
+			agent, err = key.startSshAgent(logger)
-			}
+			installation.SshAgent = &agent
-			return ioutil.WriteFile(path, []byte(key.SshKey.PrivateKey+"\n"), 0600)
+
+			//err = os.WriteFile(installationPath, []byte(key.SshKey.PrivateKey+"\n"), 0600)
 		}
 	case AccessKeyRoleAnsiblePasswordVault:
 		switch key.Type {
 		case AccessKeyLoginPassword:
-			return ioutil.WriteFile(path, []byte(key.LoginPassword.Password), 0600)
+			err = os.WriteFile(installationPath, []byte(key.LoginPassword.Password), 0600)
 		}
 	case AccessKeyRoleAnsibleBecomeUser:
 		switch key.Type {
 		case AccessKeyLoginPassword:
 			content := make(map[string]string)
-			content["ansible_become_user"] = key.LoginPassword.Login
+			if len(key.LoginPassword.Login) > 0 {
+				content["ansible_become_user"] = key.LoginPassword.Login
+			}
 			content["ansible_become_password"] = key.LoginPassword.Password
 			var bytes []byte
 			bytes, err = json.Marshal(content)
 			if err != nil {
-				return err
+				return
 			}
-			return ioutil.WriteFile(path, bytes, 0600)
+			err = os.WriteFile(installationPath, bytes, 0600)
 		default:
-			return fmt.Errorf("access key type not supported for ansible user")
+			err = fmt.Errorf("access key type not supported for ansible user")
 		}
 	case AccessKeyRoleAnsibleUser:
 		switch key.Type {
 		case AccessKeySSH:
-			if key.SshKey.Passphrase != "" {
+			var agent lib.SshAgent
-				return fmt.Errorf("ssh key with passphrase not supported")
+			agent, err = key.startSshAgent(logger)
-			}
+			installation.SshAgent = &agent
-			return ioutil.WriteFile(path, []byte(key.SshKey.PrivateKey+"\n"), 0600)
+			//err = os.WriteFile(installationPath, []byte(key.SshKey.PrivateKey+"\n"), 0600)
 		case AccessKeyLoginPassword:
 			content := make(map[string]string)
 			content["ansible_user"] = key.LoginPassword.Login
@@ -127,33 +169,19 @@ func (key *AccessKey) Install(usage AccessKeyRole) error {
 			var bytes []byte
 			bytes, err = json.Marshal(content)
 			if err != nil {
-				return err
+				return
 			}
-			return ioutil.WriteFile(path, bytes, 0600)
+			err = os.WriteFile(installationPath, bytes, 0600)
 
 		default:
-			return fmt.Errorf("access key type not supported for ansible user")
+			err = fmt.Errorf("access key type not supported for ansible user")
 		}
 	}
 
-	return nil
+	return
 }
 
-func (key AccessKey) Destroy() error {
+func (key *AccessKey) Validate(validateSecretFields bool) error {
-	path := key.GetPath()
-	_, err := os.Stat(path)
-	if os.IsNotExist(err) {
-		return nil
-	}
-	return os.Remove(path)
-}
-
-// GetPath returns the location of the access key once written to disk
-func (key AccessKey) GetPath() string {
-	return util.Config.TmpPath + "/access_key_" + strconv.FormatInt(key.InstallationKey, 10)
-}
-
-func (key AccessKey) Validate(validateSecretFields bool) error {
 	if key.Name == "" {
 		return fmt.Errorf("name can not be empty")
 	}
@@ -198,7 +226,7 @@ func (key *AccessKey) SerializeSecret() error {
 		return fmt.Errorf("invalid access token type")
 	}
 
-	encryptionString := util.Config.GetAccessKeyEncryption()
+	encryptionString := util.Config.AccessKeyEncryption
 
 	if encryptionString == "" {
 		secret := base64.StdEncoding.EncodeToString(plaintext)
@@ -251,13 +279,11 @@ func (key *AccessKey) unmarshalAppropriateField(secret []byte) (err error) {
 	return
 }
 
-//func (key *AccessKey) ClearSecret() {
-//	key.LoginPassword = LoginPassword{}
-//	key.SshKey = SshKey{}
-//	key.PAT = ""
-//}
-
 func (key *AccessKey) DeserializeSecret() error {
+	return key.DeserializeSecret2(util.Config.AccessKeyEncryption)
+}
+
+func (key *AccessKey) DeserializeSecret2(encryptionString string) error {
 	if key.Secret == nil || *key.Secret == "" {
 		return nil
 	}
@@ -279,12 +305,10 @@ func (key *AccessKey) DeserializeSecret() error {
 		return err
 	}
 
-	encryptionString := util.Config.GetAccessKeyEncryption()
-
 	if encryptionString == "" {
 		err = key.unmarshalAppropriateField(ciphertext)
 		if _, ok := err.(*json.SyntaxError); ok {
-			err = fmt.Errorf("secret must be valid json")
+			err = fmt.Errorf("secret must be valid json in key '%s'", key.Name)
 		}
 		return err
 	}

db/BackupEntity.go

@@ -0,0 +1,54 @@
+package db
+
+type BackupEntity interface {
+	GetID() int
+	GetName() string
+}
+
+func (e View) GetID() int {
+	return e.ID
+}
+
+func (e View) GetName() string {
+	return e.Title
+}
+
+func (e Template) GetID() int {
+	return e.ID
+}
+
+func (e Template) GetName() string {
+	return e.Name
+}
+
+func (e Inventory) GetID() int {
+	return e.ID
+}
+
+func (e Inventory) GetName() string {
+	return e.Name
+}
+
+func (e AccessKey) GetID() int {
+	return e.ID
+}
+
+func (e AccessKey) GetName() string {
+	return e.Name
+}
+
+func (e Repository) GetID() int {
+	return e.ID
+}
+
+func (e Repository) GetName() string {
+	return e.Name
+}
+
+func (e Environment) GetID() int {
+	return e.ID
+}
+
+func (e Environment) GetName() string {
+	return e.Name
+}

db/Event.go

@@ -6,9 +6,11 @@ import (
 
 // Event represents information generated by ansible or api action captured to the database during execution
 type Event struct {
-	ID          int              `db:"id" json:"-"`
+	ID            int  `db:"id" json:"-"`
-	UserID      *int             `db:"user_id" json:"user_id"`
+	UserID        *int `db:"user_id" json:"user_id"`
-	ProjectID   *int             `db:"project_id" json:"project_id"`
+	ProjectID     *int `db:"project_id" json:"project_id"`
+	IntegrationID *int `db:"integration_id" json:"integration_id"`
+
 	ObjectID    *int             `db:"object_id" json:"object_id"`
 	ObjectType  *EventObjectType `db:"object_type" json:"object_type"`
 	Description *string          `db:"description" json:"description"`
@@ -22,16 +24,19 @@ type Event struct {
 type EventObjectType string
 
 const (
-	EventTask        EventObjectType = "task"
+	EventTask                    EventObjectType = "task"
-	EventEnvironment EventObjectType = "environment"
+	EventEnvironment             EventObjectType = "environment"
-	EventInventory   EventObjectType = "inventory"
+	EventInventory               EventObjectType = "inventory"
-	EventKey         EventObjectType = "key"
+	EventKey                     EventObjectType = "key"
-	EventProject     EventObjectType = "project"
+	EventProject                 EventObjectType = "project"
-	EventRepository  EventObjectType = "repository"
+	EventRepository              EventObjectType = "repository"
-	EventSchedule    EventObjectType = "schedule"
+	EventSchedule                EventObjectType = "schedule"
-	EventTemplate    EventObjectType = "template"
+	EventTemplate                EventObjectType = "template"
-	EventUser        EventObjectType = "user"
+	EventUser                    EventObjectType = "user"
-	EventView        EventObjectType = "view"
+	EventView                    EventObjectType = "view"
+	EventIntegration             EventObjectType = "integration"
+	EventIntegrationExtractValue EventObjectType = "integrationextractvalue"
+	EventIntegrationMatcher      EventObjectType = "integrationmatcher"
 )
 
 func FillEvents(d Store, events []Event) (err error) {

db/Integration.go

@@ -0,0 +1,183 @@
+package db
+
+import (
+	"strconv"
+	"strings"
+)
+
+type IntegrationAuthMethod string
+
+const (
+	IntegrationAuthNone  = ""
+	IntegrationAuthToken = "token"
+	IntegrationAuthHmac  = "hmac"
+)
+
+type IntegrationMatchType string
+
+const (
+	IntegrationMatchHeader IntegrationMatchType = "header"
+	IntegrationMatchBody   IntegrationMatchType = "body"
+)
+
+type IntegrationMatchMethodType string
+
+const (
+	IntegrationMatchMethodEquals   IntegrationMatchMethodType = "equals"
+	IntegrationMatchMethodUnEquals IntegrationMatchMethodType = "unequals"
+	IntegrationMatchMethodContains IntegrationMatchMethodType = "contains"
+)
+
+type IntegrationBodyDataType string
+
+const (
+	IntegrationBodyDataJSON   IntegrationBodyDataType = "json"
+	IntegrationBodyDataXML    IntegrationBodyDataType = "xml"
+	IntegrationBodyDataString IntegrationBodyDataType = "string"
+)
+
+type IntegrationMatcher struct {
+	ID            int                        `db:"id" json:"id"`
+	Name          string                     `db:"name" json:"name"`
+	IntegrationID int                        `db:"integration_id" json:"integration_id"`
+	MatchType     IntegrationMatchType       `db:"match_type" json:"match_type"`
+	Method        IntegrationMatchMethodType `db:"method" json:"method"`
+	BodyDataType  IntegrationBodyDataType    `db:"body_data_type" json:"body_data_type"`
+	Key           string                     `db:"key" json:"key"`
+	Value         string                     `db:"value" json:"value"`
+}
+
+type IntegrationExtractValueSource string
+
+const (
+	IntegrationExtractBodyValue   IntegrationExtractValueSource = "body"
+	IntegrationExtractHeaderValue IntegrationExtractValueSource = "header"
+)
+
+type IntegrationExtractValue struct {
+	ID            int                           `db:"id" json:"id"`
+	Name          string                        `db:"name" json:"name"`
+	IntegrationID int                           `db:"integration_id" json:"integration_id"`
+	ValueSource   IntegrationExtractValueSource `db:"value_source" json:"value_source"`
+	BodyDataType  IntegrationBodyDataType       `db:"body_data_type" json:"body_data_type"`
+	Key           string                        `db:"key" json:"key"`
+	Variable      string                        `db:"variable" json:"variable"`
+}
+
+type IntegrationAlias struct {
+	ID            int    `db:"id" json:"id"`
+	Alias         string `db:"alias" json:"alias"`
+	ProjectID     int    `db:"project_id" json:"project_id"`
+	IntegrationID *int   `db:"integration_id" json:"integration_id"`
+}
+
+type Integration struct {
+	ID           int                   `db:"id" json:"id"`
+	Name         string                `db:"name" json:"name"`
+	ProjectID    int                   `db:"project_id" json:"project_id"`
+	TemplateID   int                   `db:"template_id" json:"template_id"`
+	AuthMethod   IntegrationAuthMethod `db:"auth_method" json:"auth_method"`
+	AuthSecretID *int                  `db:"auth_secret_id" json:"auth_secret_id"`
+	AuthHeader   string                `db:"auth_header" json:"auth_header"`
+	AuthSecret   AccessKey             `db:"-" json:"-"`
+	Searchable   bool                  `db:"searchable" json:"searchable"`
+}
+
+func (env *Integration) Validate() error {
+	if env.Name == "" {
+		return &ValidationError{"No Name set for integration"}
+	}
+	return nil
+}
+
+func (env *IntegrationMatcher) Validate() error {
+	if env.MatchType == "" {
+		return &ValidationError{"No Match Type set"}
+	} else {
+		if env.Key == "" {
+			return &ValidationError{"No key set"}
+		}
+		if env.Value == "" {
+			return &ValidationError{"No value set"}
+		}
+
+	}
+
+	if env.Name == "" {
+		return &ValidationError{"No Name set for integration"}
+	}
+
+	return nil
+}
+
+func (env *IntegrationExtractValue) Validate() error {
+	if env.ValueSource == "" {
+		return &ValidationError{"No Value Source defined"}
+	}
+
+	if env.Name == "" {
+		return &ValidationError{"No Name set for integration"}
+	}
+
+	if env.ValueSource == IntegrationExtractBodyValue {
+		if env.BodyDataType == "" {
+			return &ValidationError{"Value Source but no body data type set"}
+		}
+
+		if env.BodyDataType == IntegrationBodyDataJSON {
+			if env.Key == "" {
+				return &ValidationError{"No Key set for JSON Body Data extraction."}
+			}
+		}
+	}
+
+	if env.ValueSource == IntegrationExtractHeaderValue {
+		if env.Key == "" {
+			return &ValidationError{"Value Source set but no Key set"}
+		}
+	}
+
+	return nil
+}
+
+func (matcher *IntegrationMatcher) String() string {
+	var builder strings.Builder
+	// ID:1234 body/json key == value on Extractor: 1234
+	builder.WriteString("ID:" + strconv.Itoa(matcher.ID) + " " + string(matcher.MatchType))
+
+	if matcher.MatchType == IntegrationMatchBody {
+		builder.WriteString("/" + string(matcher.BodyDataType))
+	}
+
+	builder.WriteString(" " + matcher.Key + " ")
+
+	switch matcher.Method {
+	case IntegrationMatchMethodEquals:
+		builder.WriteString("==")
+	case IntegrationMatchMethodUnEquals:
+		builder.WriteString("!=")
+	case IntegrationMatchMethodContains:
+		builder.WriteString(" contains ")
+	default:
+
+	}
+
+	builder.WriteString(matcher.Value + ", on Extractor: " + strconv.Itoa(matcher.IntegrationID))
+
+	return builder.String()
+}
+
+func (value *IntegrationExtractValue) String() string {
+	var builder strings.Builder
+
+	// ID:1234 body/json from key as argument
+	builder.WriteString("ID:" + strconv.Itoa(value.ID) + " " + string(value.ValueSource))
+
+	if value.ValueSource == IntegrationExtractBodyValue {
+		builder.WriteString("/" + string(value.BodyDataType))
+	}
+
+	builder.WriteString(" from " + value.Key + " as " + value.Variable)
+
+	return builder.String()
+}

db/Inventory.go

@@ -1,9 +1,13 @@
 package db
 
+type InventoryType string
+
 const (
-	InventoryStatic     = "static"
+	InventoryNone       InventoryType = "none"
-	InventoryStaticYaml = "static-yaml"
+	InventoryStatic     InventoryType = "static"
-	InventoryFile       = "file"
+	InventoryStaticYaml InventoryType = "static-yaml"
+	// InventoryFile means that it is path to the Ansible inventory file
+	InventoryFile InventoryType = "file"
 )
 
 // Inventory is the model of an ansible inventory file
@@ -21,7 +25,9 @@ type Inventory struct {
 	BecomeKey   AccessKey `db:"-" json:"-"`
 
 	// static/file
-	Type string `db:"type" json:"type"`
+	Type InventoryType `db:"type" json:"type"`
+
+	HolderID *int `db:"holder_id" json:"holder_id"`
 }
 
 func FillInventory(d Store, inventory *Inventory) (err error) {

db/Migration.go

@@ -59,6 +59,12 @@ func GetMigrations() []Migration {
 		{Version: "2.8.51"},
 		{Version: "2.8.57"},
 		{Version: "2.8.58"},
+		{Version: "2.8.91"},
+		{Version: "2.9.6"},
+		{Version: "2.9.46"},
+		{Version: "2.9.60"},
+		{Version: "2.9.61"},
+		{Version: "2.9.62"},
 	}
 }
 

db/Option.go

@@ -0,0 +1,6 @@
+package db
+
+type Option struct {
+	Key   string `db:"key" json:"key"`
+	Value string `db:"value" json:"value"`
+}

db/Project.go

@@ -12,4 +12,5 @@ type Project struct {
 	Alert            bool      `db:"alert" json:"alert"`
 	AlertChat        *string   `db:"alert_chat" json:"alert_chat"`
 	MaxParallelTasks int       `db:"max_parallel_tasks" json:"max_parallel_tasks"`
+	Type             string    `db:"type" json:"type"`
 }

db/ProjectUser.go

@@ -1,8 +1,47 @@
 package db
 
-type ProjectUser struct {
+type ProjectUserRole string
-	ID        int  `db:"id" json:"-"`
+
-	ProjectID int  `db:"project_id" json:"project_id"`
+const (
-	UserID    int  `db:"user_id" json:"user_id"`
+	ProjectOwner      ProjectUserRole = "owner"
-	Admin     bool `db:"admin" json:"admin"`
+	ProjectManager    ProjectUserRole = "manager"
+	ProjectTaskRunner ProjectUserRole = "task_runner"
+	ProjectGuest      ProjectUserRole = "guest"
+	ProjectNone       ProjectUserRole = ""
+)
+
+type ProjectUserPermission int
+
+const (
+	CanRunProjectTasks ProjectUserPermission = 1 << iota
+	CanUpdateProject
+	CanManageProjectResources
+	CanManageProjectUsers
+)
+
+var rolePermissions = map[ProjectUserRole]ProjectUserPermission{
+	ProjectOwner:      CanRunProjectTasks | CanManageProjectResources | CanUpdateProject | CanManageProjectUsers,
+	ProjectManager:    CanRunProjectTasks | CanManageProjectResources,
+	ProjectTaskRunner: CanRunProjectTasks,
+	ProjectGuest:      0,
+}
+
+func (r ProjectUserRole) IsValid() bool {
+	_, ok := rolePermissions[r]
+	return ok
+}
+
+type ProjectUser struct {
+	ID        int             `db:"id" json:"-"`
+	ProjectID int             `db:"project_id" json:"project_id"`
+	UserID    int             `db:"user_id" json:"user_id"`
+	Role      ProjectUserRole `db:"role" json:"role"`
+}
+
+func (r ProjectUserRole) Can(permissions ProjectUserPermission) bool {
+	return (rolePermissions[r] & permissions) == permissions
+}
+
+func (r ProjectUserRole) GetPermissions() ProjectUserPermission {
+	return rolePermissions[r]
 }

db/ProjectUser_test.go

@@ -0,0 +1,15 @@
+package db
+
+import (
+	"testing"
+)
+
+func TestProjectUsers_RoleCan(t *testing.T) {
+	if !ProjectManager.Can(CanManageProjectResources) {
+		t.Fatal()
+	}
+
+	if ProjectManager.Can(CanUpdateProject) {
+		t.Fatal()
+	}
+}

db/Runner.go

@@ -0,0 +1,17 @@
+package db
+
+type RunnerState string
+
+//const (
+//	RunnerOffline RunnerState = "offline"
+//	RunnerActive  RunnerState = "active"
+//)
+
+type Runner struct {
+	ID        int    `db:"id" json:"-"`
+	Token     string `db:"token" json:"-"`
+	ProjectID *int   `db:"project_id" json:"project_id"`
+	//State            RunnerState `db:"state" json:"state"`
+	Webhook          string `db:"webhook" json:"webhook"`
+	MaxParallelTasks int    `db:"max_parallel_tasks" json:"max_parallel_tasks"`
+}

db/Store.go

@@ -3,7 +3,7 @@ package db
 import (
 	"encoding/json"
 	"errors"
-	log "github.com/Sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	"reflect"
 	"strings"
 	"time"
@@ -51,6 +51,15 @@ type ObjectReferrers struct {
 	Repositories []ObjectReferrer `json:"repositories"`
 }
 
+type IntegrationReferrers struct {
+	IntegrationMatchers      []ObjectReferrer `json:"matchers"`
+	IntegrationExtractValues []ObjectReferrer `json:"values"`
+}
+
+type IntegrationExtractorChildReferrers struct {
+	Integrations []ObjectReferrer `json:"integrations"`
+}
+
 // ObjectProps describe database entities.
 // It mainly used for NoSQL implementations (currently BoltDB) to preserve same
 // data structure of different implementations and easy change it if required.
@@ -100,6 +109,9 @@ type Store interface {
 	// if a rollback exists
 	TryRollbackMigration(version Migration)
 
+	GetOption(key string) (string, error)
+	SetOption(key string, value string) error
+
 	GetEnvironment(projectID int, environmentID int) (Environment, error)
 	GetEnvironmentRefs(projectID int, environmentID int) (ObjectReferrers, error)
 	GetEnvironments(projectID int, params RetrieveQueryParams) ([]Environment, error)
@@ -124,6 +136,34 @@ type Store interface {
 	GetAccessKey(projectID int, accessKeyID int) (AccessKey, error)
 	GetAccessKeyRefs(projectID int, accessKeyID int) (ObjectReferrers, error)
 	GetAccessKeys(projectID int, params RetrieveQueryParams) ([]AccessKey, error)
+	RekeyAccessKeys(oldKey string) error
+
+	CreateIntegration(integration Integration) (newIntegration Integration, err error)
+	GetIntegrations(projectID int, params RetrieveQueryParams) ([]Integration, error)
+	GetIntegration(projectID int, integrationID int) (integration Integration, err error)
+	UpdateIntegration(integration Integration) error
+	GetIntegrationRefs(projectID int, integrationID int) (IntegrationReferrers, error)
+	DeleteIntegration(projectID int, integrationID int) error
+
+	CreateIntegrationExtractValue(projectId int, value IntegrationExtractValue) (newValue IntegrationExtractValue, err error)
+	GetIntegrationExtractValues(projectID int, params RetrieveQueryParams, integrationID int) ([]IntegrationExtractValue, error)
+	GetIntegrationExtractValue(projectID int, valueID int, integrationID int) (value IntegrationExtractValue, err error)
+	UpdateIntegrationExtractValue(projectID int, integrationExtractValue IntegrationExtractValue) error
+	GetIntegrationExtractValueRefs(projectID int, valueID int, integrationID int) (IntegrationExtractorChildReferrers, error)
+	DeleteIntegrationExtractValue(projectID int, valueID int, integrationID int) error
+
+	CreateIntegrationMatcher(projectID int, matcher IntegrationMatcher) (newMatcher IntegrationMatcher, err error)
+	GetIntegrationMatchers(projectID int, params RetrieveQueryParams, integrationID int) ([]IntegrationMatcher, error)
+	GetIntegrationMatcher(projectID int, matcherID int, integrationID int) (matcher IntegrationMatcher, err error)
+	UpdateIntegrationMatcher(projectID int, integrationMatcher IntegrationMatcher) error
+	GetIntegrationMatcherRefs(projectID int, matcherID int, integrationID int) (IntegrationExtractorChildReferrers, error)
+	DeleteIntegrationMatcher(projectID int, matcherID int, integrationID int) error
+
+	CreateIntegrationAlias(alias IntegrationAlias) (IntegrationAlias, error)
+	GetIntegrationAlias(projectID int, integrationID *int) (IntegrationAlias, error)
+	GetIntegrationAliasByAlias(alias string) (IntegrationAlias, error)
+	UpdateIntegrationAlias(alias IntegrationAlias) error
+	DeleteIntegrationAlias(projectID int, integrationID *int) error
 
 	UpdateAccessKey(accessKey AccessKey) error
 	CreateAccessKey(accessKey AccessKey) (AccessKey, error)
@@ -142,6 +182,7 @@ type Store interface {
 	GetUserByLoginOrEmail(login string, email string) (User, error)
 
 	GetProject(projectID int) (Project, error)
+	GetAllProjects() ([]Project, error)
 	GetProjects(userID int) ([]Project, error)
 	CreateProject(project Project) (Project, error)
 	DeleteProject(projectID int) error
@@ -162,7 +203,7 @@ type Store interface {
 	GetSchedule(projectID int, scheduleID int) (Schedule, error)
 	DeleteSchedule(projectID int, scheduleID int) error
 
-	GetProjectUsers(projectID int, params RetrieveQueryParams) ([]User, error)
+	GetProjectUsers(projectID int, params RetrieveQueryParams) ([]UserWithProjectRole, error)
 	CreateProjectUser(projectUser ProjectUser) (ProjectUser, error)
 	DeleteProjectUser(projectID int, userID int) error
 	GetProjectUser(projectID int, userID int) (ProjectUser, error)
@@ -199,6 +240,15 @@ type Store interface {
 	CreateView(view View) (View, error)
 	DeleteView(projectID int, viewID int) error
 	SetViewPositions(projectID int, viewPositions map[int]int) error
+
+	GetRunner(projectID int, runnerID int) (Runner, error)
+	GetRunners(projectID int) ([]Runner, error)
+	DeleteRunner(projectID int, runnerID int) error
+	GetGlobalRunner(runnerID int) (Runner, error)
+	GetGlobalRunners() ([]Runner, error)
+	DeleteGlobalRunner(runnerID int) error
+	UpdateRunner(runner Runner) error
+	CreateRunner(runner Runner) (Runner, error)
 }
 
 var AccessKeyProps = ObjectProps{
@@ -210,6 +260,37 @@ var AccessKeyProps = ObjectProps{
 	DefaultSortingColumn:  "name",
 }
 
+var IntegrationProps = ObjectProps{
+	TableName:             "project__integration",
+	Type:                  reflect.TypeOf(Integration{}),
+	PrimaryColumnName:     "id",
+	ReferringColumnSuffix: "integration_id",
+	SortableColumns:       []string{"name"},
+	DefaultSortingColumn:  "name",
+}
+
+var IntegrationExtractValueProps = ObjectProps{
+	TableName:            "project__integration_extract_value",
+	Type:                 reflect.TypeOf(IntegrationExtractValue{}),
+	PrimaryColumnName:    "id",
+	SortableColumns:      []string{"name"},
+	DefaultSortingColumn: "name",
+}
+
+var IntegrationMatcherProps = ObjectProps{
+	TableName:            "project__integration_matcher",
+	Type:                 reflect.TypeOf(IntegrationMatcher{}),
+	PrimaryColumnName:    "id",
+	SortableColumns:      []string{"name"},
+	DefaultSortingColumn: "name",
+}
+
+var IntegrationAliasProps = ObjectProps{
+	TableName:         "project__integration_alias",
+	Type:              reflect.TypeOf(IntegrationAlias{}),
+	PrimaryColumnName: "id",
+}
+
 var EnvironmentProps = ObjectProps{
 	TableName:             "project__environment",
 	Type:                  reflect.TypeOf(Environment{}),
@@ -258,11 +339,12 @@ var ProjectUserProps = ObjectProps{
 }
 
 var ProjectProps = ObjectProps{
-	TableName:            "project",
+	TableName:             "project",
-	Type:                 reflect.TypeOf(Project{}),
+	Type:                  reflect.TypeOf(Project{}),
-	PrimaryColumnName:    "id",
+	PrimaryColumnName:     "id",
-	DefaultSortingColumn: "name",
+	ReferringColumnSuffix: "project_id",
-	IsGlobal:             true,
+	DefaultSortingColumn:  "name",
+	IsGlobal:              true,
 }
 
 var UserProps = ObjectProps{
@@ -304,6 +386,20 @@ var ViewProps = ObjectProps{
 	DefaultSortingColumn: "position",
 }
 
+var GlobalRunnerProps = ObjectProps{
+	TableName:         "runner",
+	Type:              reflect.TypeOf(Runner{}),
+	PrimaryColumnName: "id",
+	IsGlobal:          true,
+}
+
+var OptionProps = ObjectProps{
+	TableName:         "option",
+	Type:              reflect.TypeOf(Option{}),
+	PrimaryColumnName: "key",
+	IsGlobal:          true,
+}
+
 func (p ObjectProps) GetReferringFieldsFrom(t reflect.Type) (fields []string, err error) {
 	n := t.NumField()
 	for i := 0; i < n; i++ {

db/Task.go

@@ -1,29 +1,19 @@
 package db
 
 import (
+	"github.com/ansible-semaphore/semaphore/lib"
 	"time"
 )
 
-type TaskStatus string
+// Task is a model of a task which will be executed by the runner
-
-const (
-	TaskRunningStatus  TaskStatus = "running"
-	TaskWaitingStatus  TaskStatus = "waiting"
-	TaskStoppingStatus TaskStatus = "stopping"
-	TaskStoppedStatus  TaskStatus = "stopped"
-	TaskSuccessStatus  TaskStatus = "success"
-	TaskFailStatus     TaskStatus = "error"
-)
-
-//Task is a model of a task which will be executed by the runner
 type Task struct {
 	ID         int `db:"id" json:"id"`
 	TemplateID int `db:"template_id" json:"template_id" binding:"required"`
 	ProjectID  int `db:"project_id" json:"project_id"`
 
-	Status TaskStatus `db:"status" json:"status"`
+	Status lib.TaskStatus `db:"status" json:"status"`
-	Debug  bool       `db:"debug" json:"debug"`
 
+	Debug  bool `db:"debug" json:"debug"`
 	DryRun bool `db:"dry_run" json:"dry_run"`
 	Diff   bool `db:"diff" json:"diff"`
 
@@ -54,6 +44,8 @@ type Task struct {
 	Version *string `db:"version" json:"version"`
 
 	Arguments *string `db:"arguments" json:"arguments"`
+
+	InventoryID *int `db:"inventory_id" json:"inventory_id"`
 }
 
 func (task *Task) GetIncomingVersion(d Store) *string {

db/Template.go

@@ -12,6 +12,12 @@ const (
 	TemplateDeploy TemplateType = "deploy"
 )
 
+type TemplateApp string
+
+const (
+	TemplateAnsible = ""
+)
+
 type SurveyVarType string
 
 const (
@@ -73,6 +79,8 @@ type Template struct {
 	SurveyVars     []SurveyVar `db:"-" json:"survey_vars"`
 
 	SuppressSuccessAlerts bool `db:"suppress_success_alerts" json:"suppress_success_alerts"`
+
+	App TemplateApp `db:"app" json:"app"`
 }
 
 func (tpl *Template) Validate() error {
@@ -124,6 +132,15 @@ func FillTemplate(d Store, template *Template) (err error) {
 		return
 	}
 
+	var tasks []TaskWithTpl
+	tasks, err = d.GetTemplateTasks(template.ProjectID, template.ID, RetrieveQueryParams{Count: 1})
+	if err != nil {
+		return
+	}
+	if len(tasks) > 0 {
+		template.LastTask = &tasks[0]
+	}
+
 	if template.SurveyVarsJSON != nil {
 		err = json.Unmarshal([]byte(*template.SurveyVarsJSON), &template.SurveyVars)
 	}

db/User.go

@@ -17,6 +17,11 @@ type User struct {
 	Alert    bool      `db:"alert" json:"alert"`
 }
 
+type UserWithProjectRole struct {
+	Role ProjectUserRole `db:"role" json:"role"`
+	User
+}
+
 // UserWithPwd extends User structure with field for unhashed password received from JSON.
 type UserWithPwd struct {
 	Pwd string `db:"-" json:"password"` // unhashed password from JSON

db/bolt/BoltDb.go

@@ -82,6 +82,7 @@ func (d *BoltDb) Connect(token string) {
 	}
 
 	if _, exists := d.connections[token]; exists {
+		// Use for debugging
 		panic(fmt.Errorf("Connection " + token + " already exists"))
 	}
 
@@ -120,6 +121,7 @@ func (d *BoltDb) Close(token string) {
 	_, exists := d.connections[token]
 
 	if !exists {
+		// Use for debugging
 		panic(fmt.Errorf("can not close closed connection " + token))
 	}
 
@@ -360,16 +362,20 @@ func unmarshalObjects(rawData enumerable, props db.ObjectProps, params db.Retrie
 	return
 }
 
+func (d *BoltDb) getObjectsTx(tx *bbolt.Tx, bucketID int, props db.ObjectProps, params db.RetrieveQueryParams, filter func(interface{}) bool, objects interface{}) error {
+	b := tx.Bucket(makeBucketId(props, bucketID))
+	var c enumerable
+	if b == nil {
+		c = emptyEnumerable{}
+	} else {
+		c = b.Cursor()
+	}
+	return unmarshalObjects(c, props, params, filter, objects)
+}
+
 func (d *BoltDb) getObjects(bucketID int, props db.ObjectProps, params db.RetrieveQueryParams, filter func(interface{}) bool, objects interface{}) error {
 	return d.db.View(func(tx *bbolt.Tx) error {
-		b := tx.Bucket(makeBucketId(props, bucketID))
+		return d.getObjectsTx(tx, bucketID, props, params, filter, objects)
-		var c enumerable
-		if b == nil {
-			c = emptyEnumerable{}
-		} else {
-			c = b.Cursor()
-		}
-		return unmarshalObjects(c, props, params, filter, objects)
 	})
 }
 
@@ -399,23 +405,83 @@ func (d *BoltDb) deleteObject(bucketID int, props db.ObjectProps, objectID objec
 	return d.db.Update(fn)
 }
 
+func (d *BoltDb) updateObjectTx(tx *bbolt.Tx, bucketID int, props db.ObjectProps, object interface{}) error {
+	b := tx.Bucket(makeBucketId(props, bucketID))
+	if b == nil {
+		return db.ErrNotFound
+	}
+
+	idFieldName, err := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
+
+	if err != nil {
+		return err
+	}
+
+	idValue := reflect.ValueOf(object).FieldByName(idFieldName)
+
+	var objID objectID
+
+	switch idValue.Kind() {
+	case reflect.Int,
+		reflect.Int8,
+		reflect.Int16,
+		reflect.Int32,
+		reflect.Int64,
+		reflect.Uint,
+		reflect.Uint8,
+		reflect.Uint16,
+		reflect.Uint32,
+		reflect.Uint64:
+		objID = intObjectID(idValue.Int())
+	case reflect.String:
+		objID = strObjectID(idValue.String())
+	}
+
+	if objID == nil {
+		return fmt.Errorf("unsupported ID type")
+	}
+
+	if b.Get(objID.ToBytes()) == nil {
+		return db.ErrNotFound
+	}
+
+	str, err := marshalObject(object)
+	if err != nil {
+		return err
+	}
+
+	return b.Put(objID.ToBytes(), str)
+}
+
 // updateObject updates data for object in database.
 func (d *BoltDb) updateObject(bucketID int, props db.ObjectProps, object interface{}) error {
 	return d.db.Update(func(tx *bbolt.Tx) error {
-		b := tx.Bucket(makeBucketId(props, bucketID))
+		return d.updateObjectTx(tx, bucketID, props, object)
-		if b == nil {
+	})
-			return db.ErrNotFound
+}
+
+func (d *BoltDb) createObjectTx(tx *bbolt.Tx, bucketID int, props db.ObjectProps, object interface{}) (interface{}, error) {
+	b, err := tx.CreateBucketIfNotExists(makeBucketId(props, bucketID))
+
+	if err != nil {
+		return nil, err
+	}
+
+	objPtr := reflect.ValueOf(&object).Elem()
+
+	tmpObj := reflect.New(objPtr.Elem().Type()).Elem()
+	tmpObj.Set(objPtr.Elem())
+
+	var objID objectID
+
+	if props.PrimaryColumnName != "" {
+		idFieldName, err2 := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
+
+		if err2 != nil {
+			return nil, err2
 		}
 
-		idFieldName, err := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
+		idValue := tmpObj.FieldByName(idFieldName)
-
-		if err != nil {
-			return err
-		}
-
-		idValue := reflect.ValueOf(object).FieldByName(idFieldName)
-
-		var objID objectID
 
 		switch idValue.Kind() {
 		case reflect.Int,
@@ -428,114 +494,101 @@ func (d *BoltDb) updateObject(bucketID int, props db.ObjectProps, object interfa
 			reflect.Uint16,
 			reflect.Uint32,
 			reflect.Uint64:
-			objID = intObjectID(idValue.Int())
+			if idValue.Int() == 0 {
-		case reflect.String:
-			objID = strObjectID(idValue.String())
-		}
-
-		if objID == nil {
-			return fmt.Errorf("unsupported ID type")
-		}
-
-		if b.Get(objID.ToBytes()) == nil {
-			return db.ErrNotFound
-		}
-
-		str, err := marshalObject(object)
-		if err != nil {
-			return err
-		}
-
-		return b.Put(objID.ToBytes(), str)
-	})
-}
-
-func (d *BoltDb) createObject(bucketID int, props db.ObjectProps, object interface{}) (interface{}, error) {
-	err := d.db.Update(func(tx *bbolt.Tx) error {
-		b, err := tx.CreateBucketIfNotExists(makeBucketId(props, bucketID))
-
-		if err != nil {
-			return err
-		}
-
-		objPtr := reflect.ValueOf(&object).Elem()
-
-		tmpObj := reflect.New(objPtr.Elem().Type()).Elem()
-		tmpObj.Set(objPtr.Elem())
-
-		var objID objectID
-
-		if props.PrimaryColumnName != "" {
-			idFieldName, err2 := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
-
-			if err2 != nil {
-				return err2
-			}
-
-			idValue := tmpObj.FieldByName(idFieldName)
-
-			switch idValue.Kind() {
-			case reflect.Int,
-				reflect.Int8,
-				reflect.Int16,
-				reflect.Int32,
-				reflect.Int64,
-				reflect.Uint,
-				reflect.Uint8,
-				reflect.Uint16,
-				reflect.Uint32,
-				reflect.Uint64:
-				if idValue.Int() == 0 {
-					id, err3 := b.NextSequence()
-					if err3 != nil {
-						return err3
-					}
-					if props.SortInverted {
-						id = MaxID - id
-					}
-					idValue.SetInt(int64(id))
-				}
-
-				objID = intObjectID(idValue.Int())
-			case reflect.String:
-				if idValue.String() == "" {
-					return fmt.Errorf("object ID can not be empty string")
-				}
-				objID = strObjectID(idValue.String())
-			case reflect.Invalid:
 				id, err3 := b.NextSequence()
 				if err3 != nil {
-					return err3
+					return nil, err3
 				}
-				objID = intObjectID(id)
+				if props.SortInverted {
-			default:
+					id = MaxID - id
-				return fmt.Errorf("unsupported ID type")
+				}
+				idValue.SetInt(int64(id))
 			}
-		} else {
+
-			id, err2 := b.NextSequence()
+			objID = intObjectID(idValue.Int())
-			if err2 != nil {
+		case reflect.String:
-				return err2
+			if idValue.String() == "" {
+				return nil, fmt.Errorf("object ID can not be empty string")
 			}
-			if props.SortInverted {
+			objID = strObjectID(idValue.String())
-				id = MaxID - id
+		case reflect.Invalid:
+			id, err3 := b.NextSequence()
+			if err3 != nil {
+				return nil, err3
 			}
 			objID = intObjectID(id)
+		default:
+			return nil, fmt.Errorf("unsupported ID type")
 		}
-
+	} else {
-		if objID == nil {
+		id, err2 := b.NextSequence()
-			return fmt.Errorf("object ID can not be nil")
+		if err2 != nil {
+			return nil, err2
 		}
-
+		if props.SortInverted {
-		objPtr.Set(tmpObj)
+			id = MaxID - id
-		str, err := marshalObject(object)
-		if err != nil {
-			return err
 		}
+		objID = intObjectID(id)
+	}
 
-		return b.Put(objID.ToBytes(), str)
+	if objID == nil {
+		return nil, fmt.Errorf("object ID can not be nil")
+	}
+
+	objPtr.Set(tmpObj)
+	str, err := marshalObject(object)
+	if err != nil {
+		return nil, err
+	}
+
+	return object, b.Put(objID.ToBytes(), str)
+}
+
+func (d *BoltDb) createObject(bucketID int, props db.ObjectProps, object interface{}) (res interface{}, err error) {
+
+	_ = d.db.Update(func(tx *bbolt.Tx) error {
+		res, err = d.createObjectTx(tx, bucketID, props, object)
+		return err
 	})
 
-	return object, err
+	return
+}
+
+func (d *BoltDb) getIntegrationRefs(projectID int, objectProps db.ObjectProps, objectID int) (refs db.IntegrationReferrers, err error) {
+	//refs.IntegrationExtractors, err = d.getReferringObjectByParentID(projectID, objectProps, objectID, db.IntegrationExtractorProps)
+
+	return
+}
+
+func (d *BoltDb) getIntegrationExtractorChildrenRefs(integrationID int, objectProps db.ObjectProps, objectID int) (refs db.IntegrationExtractorChildReferrers, err error) {
+	//refs.IntegrationExtractors, err = d.getReferringObjectByParentID(objectID, objectProps, integrationID, db.IntegrationExtractorProps)
+	//if err != nil {
+	//	return
+	//}
+
+	return
+}
+
+func (d *BoltDb) getReferringObjectByParentID(parentID int, objProps db.ObjectProps, objID int, referringObjectProps db.ObjectProps) (referringObjs []db.ObjectReferrer, err error) {
+	referringObjs = make([]db.ObjectReferrer, 0)
+
+	var referringObjectOfType reflect.Value = reflect.New(reflect.SliceOf(referringObjectProps.Type))
+	err = d.getObjects(parentID, referringObjectProps, db.RetrieveQueryParams{}, func(referringObj interface{}) bool {
+		return isObjectReferredBy(objProps, intObjectID(objID), referringObj)
+	}, referringObjectOfType.Interface())
+
+	if err != nil {
+		return
+	}
+
+	for i := 0; i < referringObjectOfType.Elem().Len(); i++ {
+		referringObjs = append(referringObjs, db.ObjectReferrer{
+			ID:   int(referringObjectOfType.Elem().Index(i).FieldByName("ID").Int()),
+			Name: referringObjectOfType.Elem().Index(i).FieldByName("Name").String(),
+		})
+	}
+
+	return
 }
 
 func (d *BoltDb) getObjectRefs(projectID int, objectProps db.ObjectProps, objectID int) (refs db.ObjectReferrers, err error) {

db/bolt/access_key.go

@@ -2,6 +2,7 @@ package bolt
 
 import (
 	"github.com/ansible-semaphore/semaphore/db"
+	"go.etcd.io/bbolt"
 )
 
 func (d *BoltDb) GetAccessKey(projectID int, accessKeyID int) (key db.AccessKey, err error) {
@@ -59,3 +60,43 @@ func (d *BoltDb) CreateAccessKey(key db.AccessKey) (db.AccessKey, error) {
 func (d *BoltDb) DeleteAccessKey(projectID int, accessKeyID int) error {
 	return d.deleteObject(projectID, db.AccessKeyProps, intObjectID(accessKeyID), nil)
 }
+
+func (d *BoltDb) RekeyAccessKeys(oldKey string) error {
+	return d.db.Update(func(tx *bbolt.Tx) error {
+		var allProjects []db.Project
+
+		err := d.getObjectsTx(tx, 0, db.ProjectProps, db.RetrieveQueryParams{}, nil, &allProjects)
+
+		if err != nil {
+			return err
+		}
+
+		for _, project := range allProjects {
+			var keys []db.AccessKey
+			err = d.getObjectsTx(tx, project.ID, db.AccessKeyProps, db.RetrieveQueryParams{}, nil, &keys)
+			if err != nil {
+				return err
+			}
+
+			for _, key := range keys {
+				err = key.DeserializeSecret2(oldKey)
+
+				if err != nil {
+					return err
+				}
+
+				err = key.SerializeSecret()
+				if err != nil {
+					return err
+				}
+
+				err = d.updateObjectTx(tx, *key.ProjectID, db.AccessKeyProps, key)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+}

db/bolt/integrations.go

@@ -0,0 +1,279 @@
+package bolt
+
+import (
+	"github.com/ansible-semaphore/semaphore/db"
+	"go.etcd.io/bbolt"
+	"reflect"
+)
+
+/*
+Integrations
+*/
+func (d *BoltDb) CreateIntegration(integration db.Integration) (db.Integration, error) {
+	err := integration.Validate()
+
+	if err != nil {
+		return db.Integration{}, err
+	}
+
+	newIntegration, err := d.createObject(integration.ProjectID, db.IntegrationProps, integration)
+	return newIntegration.(db.Integration), err
+}
+
+func (d *BoltDb) GetIntegrations(projectID int, params db.RetrieveQueryParams) (integrations []db.Integration, err error) {
+	err = d.getObjects(projectID, db.IntegrationProps, params, nil, &integrations)
+	return integrations, err
+}
+
+func (d *BoltDb) GetIntegration(projectID int, integrationID int) (integration db.Integration, err error) {
+	err = d.getObject(projectID, db.IntegrationProps, intObjectID(integrationID), &integration)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (d *BoltDb) UpdateIntegration(integration db.Integration) error {
+	err := integration.Validate()
+
+	if err != nil {
+		return err
+	}
+
+	return d.updateObject(integration.ProjectID, db.IntegrationProps, integration)
+
+}
+
+func (d *BoltDb) GetIntegrationRefs(projectID int, integrationID int) (db.IntegrationReferrers, error) {
+	//return d.getObjectRefs(projectID, db.IntegrationProps, integrationID)
+	return db.IntegrationReferrers{}, nil
+}
+
+func (d *BoltDb) DeleteIntegrationExtractValue(projectID int, valueID int, integrationID int) error {
+	return d.deleteObject(projectID, db.IntegrationExtractValueProps, intObjectID(valueID), nil)
+}
+
+func (d *BoltDb) CreateIntegrationExtractValue(projectId int, value db.IntegrationExtractValue) (db.IntegrationExtractValue, error) {
+	err := value.Validate()
+
+	if err != nil {
+		return db.IntegrationExtractValue{}, err
+	}
+
+	newValue, err := d.createObject(projectId, db.IntegrationExtractValueProps, value)
+	return newValue.(db.IntegrationExtractValue), err
+
+}
+
+func (d *BoltDb) GetIntegrationExtractValues(projectID int, params db.RetrieveQueryParams, integrationID int) (values []db.IntegrationExtractValue, err error) {
+	values = make([]db.IntegrationExtractValue, 0)
+	var allValues []db.IntegrationExtractValue
+
+	err = d.getObjects(projectID, db.IntegrationExtractValueProps, params, nil, &allValues)
+
+	if err != nil {
+		return
+	}
+
+	for _, v := range allValues {
+		if v.IntegrationID == integrationID {
+			values = append(values, v)
+		}
+	}
+
+	return
+}
+
+func (d *BoltDb) GetIntegrationExtractValue(projectID int, valueID int, integrationID int) (value db.IntegrationExtractValue, err error) {
+	err = d.getObject(projectID, db.IntegrationExtractValueProps, intObjectID(valueID), &value)
+	return value, err
+}
+
+func (d *BoltDb) UpdateIntegrationExtractValue(projectID int, integrationExtractValue db.IntegrationExtractValue) error {
+	err := integrationExtractValue.Validate()
+
+	if err != nil {
+		return err
+	}
+
+	return d.updateObject(projectID, db.IntegrationExtractValueProps, integrationExtractValue)
+}
+
+func (d *BoltDb) GetIntegrationExtractValueRefs(projectID int, valueID int, integrationID int) (db.IntegrationExtractorChildReferrers, error) {
+	return d.getIntegrationExtractorChildrenRefs(projectID, db.IntegrationExtractValueProps, valueID)
+}
+
+/*
+Integration Matcher
+*/
+func (d *BoltDb) CreateIntegrationMatcher(projectID int, matcher db.IntegrationMatcher) (db.IntegrationMatcher, error) {
+	err := matcher.Validate()
+
+	if err != nil {
+		return db.IntegrationMatcher{}, err
+	}
+	newMatcher, err := d.createObject(projectID, db.IntegrationMatcherProps, matcher)
+	return newMatcher.(db.IntegrationMatcher), err
+}
+
+func (d *BoltDb) GetIntegrationMatchers(projectID int, params db.RetrieveQueryParams, integrationID int) (matchers []db.IntegrationMatcher, err error) {
+	matchers = make([]db.IntegrationMatcher, 0)
+	var allMatchers []db.IntegrationMatcher
+
+	err = d.getObjects(projectID, db.IntegrationMatcherProps, db.RetrieveQueryParams{}, nil, &allMatchers)
+
+	if err != nil {
+		return
+	}
+
+	for _, v := range allMatchers {
+		if v.IntegrationID == integrationID {
+			matchers = append(matchers, v)
+		}
+	}
+
+	return
+}
+
+func (d *BoltDb) GetIntegrationMatcher(projectID int, matcherID int, integrationID int) (matcher db.IntegrationMatcher, err error) {
+	var matchers []db.IntegrationMatcher
+	matchers, err = d.GetIntegrationMatchers(projectID, db.RetrieveQueryParams{}, integrationID)
+
+	for _, v := range matchers {
+		if v.ID == matcherID {
+			matcher = v
+		}
+	}
+
+	return
+}
+
+func (d *BoltDb) UpdateIntegrationMatcher(projectID int, integrationMatcher db.IntegrationMatcher) error {
+	err := integrationMatcher.Validate()
+
+	if err != nil {
+		return err
+	}
+
+	return d.updateObject(projectID, db.IntegrationMatcherProps, integrationMatcher)
+}
+
+func (d *BoltDb) DeleteIntegrationMatcher(projectID int, matcherID int, integrationID int) error {
+	return d.deleteObject(projectID, db.IntegrationMatcherProps, intObjectID(matcherID), nil)
+}
+func (d *BoltDb) DeleteIntegration(projectID int, integrationID int) error {
+	matchers, err := d.GetIntegrationMatchers(projectID, db.RetrieveQueryParams{}, integrationID)
+
+	if err != nil {
+		return err
+	}
+
+	for m := range matchers {
+		d.DeleteIntegrationMatcher(projectID, matchers[m].ID, integrationID)
+	}
+
+	return d.deleteObject(projectID, db.IntegrationProps, intObjectID(integrationID), nil)
+}
+
+func (d *BoltDb) GetIntegrationMatcherRefs(projectID int, matcherID int, integrationID int) (db.IntegrationExtractorChildReferrers, error) {
+	return d.getIntegrationExtractorChildrenRefs(projectID, db.IntegrationMatcherProps, matcherID)
+}
+
+var integrationAliasProps = db.ObjectProps{
+	TableName:         "integration_alias",
+	Type:              reflect.TypeOf(db.IntegrationAlias{}),
+	PrimaryColumnName: "alias",
+}
+
+var projectLevelIntegrationId = -1
+
+func (d *BoltDb) CreateIntegrationAlias(alias db.IntegrationAlias) (res db.IntegrationAlias, err error) {
+	newAlias, err := d.createObject(alias.ProjectID, db.IntegrationAliasProps, alias)
+
+	if err != nil {
+		return
+	}
+
+	res = newAlias.(db.IntegrationAlias)
+
+	_, err = d.createObject(-1, integrationAliasProps, alias)
+
+	if err != nil {
+		_ = d.DeleteIntegrationAlias(alias.ProjectID, alias.IntegrationID)
+		return
+	}
+
+	return
+}
+
+func (d *BoltDb) GetIntegrationAlias(projectID int, integrationID *int) (res db.IntegrationAlias, err error) {
+	if integrationID == nil {
+		integrationID = &projectLevelIntegrationId
+	}
+	err = d.getObject(projectID, db.IntegrationAliasProps, intObjectID(*integrationID), &res)
+	return
+}
+
+func (d *BoltDb) GetIntegrationAliasByAlias(alias string) (res db.IntegrationAlias, err error) {
+
+	err = d.getObject(-1, integrationAliasProps, strObjectID(alias), &res)
+
+	return
+}
+
+func (d *BoltDb) UpdateIntegrationAlias(alias db.IntegrationAlias) error {
+
+	var integrationID int
+	if alias.IntegrationID == nil {
+		integrationID = projectLevelIntegrationId
+	} else {
+		integrationID = *alias.IntegrationID
+	}
+
+	oldAlias, err := d.GetIntegrationAlias(alias.ProjectID, &integrationID)
+	if err != nil {
+		return err
+	}
+
+	err = d.db.Update(func(tx *bbolt.Tx) error {
+		err := d.updateObjectTx(tx, alias.ProjectID, db.IntegrationAliasProps, alias)
+		if err != nil {
+			return err
+		}
+
+		err = d.deleteObject(-1, integrationAliasProps, strObjectID(oldAlias.Alias), tx)
+		if err != nil {
+			return err
+		}
+
+		_, err = d.createObjectTx(tx, -1, integrationAliasProps, strObjectID(alias.Alias))
+
+		return err
+	})
+
+	return err
+}
+
+func (d *BoltDb) DeleteIntegrationAlias(projectID int, integrationID *int) (err error) {
+	if integrationID == nil {
+		integrationID = &projectLevelIntegrationId
+	}
+
+	alias, err := d.GetIntegrationAlias(projectID, integrationID)
+	if err != nil {
+		return
+	}
+
+	err = d.deleteObject(projectID, db.IntegrationAliasProps, intObjectID(*integrationID), nil)
+	if err != nil {
+		return
+	}
+
+	err = d.deleteObject(-1, integrationAliasProps, strObjectID(alias.Alias), nil)
+	if err != nil {
+		return
+	}
+
+	return
+}

db/bolt/migration.go

@@ -39,6 +39,8 @@ func (d *BoltDb) ApplyMigration(m db.Migration) (err error) {
 		err = migration_2_8_28{migration{d.db}}.Apply()
 	case "2.8.40":
 		err = migration_2_8_40{migration{d.db}}.Apply()
+	case "2.8.91":
+		err = migration_2_8_91{migration{d.db}}.Apply()
 	}
 
 	if err != nil {
@@ -86,8 +88,10 @@ func (d migration) getProjectIDs() (projectIDs []string, err error) {
 	return
 }
 
+// getObjects returns map of following format: map[OBJECT_ID]map[FIELD_NAME]interface{}
 func (d migration) getObjects(projectID string, objectPrefix string) (map[string]map[string]interface{}, error) {
-	repos := make(map[string]map[string]interface{})
+	repos := make(map[string]map[string]interface{}) // ???
+
 	err := d.db.View(func(tx *bbolt.Tx) error {
 		b := tx.Bucket([]byte("project__" + objectPrefix + "_" + projectID))
 		if b == nil {
@@ -99,6 +103,7 @@ func (d migration) getObjects(projectID string, objectPrefix string) (map[string
 			return json.Unmarshal(body, &r)
 		})
 	})
+
 	return repos, err
 }
 

db/bolt/migration_2_8_91.go

@@ -0,0 +1,39 @@
+package bolt
+
+type migration_2_8_91 struct {
+	migration
+}
+
+func (d migration_2_8_91) Apply() (err error) {
+	projectIDs, err := d.getProjectIDs()
+
+	if err != nil {
+		return
+	}
+
+	usersByProjectMap := make(map[string]map[string]map[string]interface{})
+
+	for _, projectID := range projectIDs {
+		usersByProjectMap[projectID], err = d.getObjects(projectID, "user")
+		if err != nil {
+			return
+		}
+	}
+
+	for projectID, projectUsers := range usersByProjectMap {
+		for userId, userData := range projectUsers {
+			if userData["admin"] == true {
+				userData["role"] = "owner"
+			} else {
+				userData["role"] = "manager"
+			}
+			delete(userData, "admin")
+			err = d.setObject(projectID, "user", userId, userData)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}

db/bolt/migration_2_8_91_test.go

@@ -0,0 +1,85 @@
+package bolt
+
+import (
+	"encoding/json"
+	"go.etcd.io/bbolt"
+	"testing"
+)
+
+func TestMigration_2_8_91_Apply(t *testing.T) {
+	store := CreateTestStore()
+
+	err := store.db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("project"))
+		if err != nil {
+			return err
+		}
+
+		err = b.Put([]byte("0000000001"), []byte("{}"))
+		if err != nil {
+			return err
+		}
+
+		r, err := tx.CreateBucketIfNotExists([]byte("project__user_0000000001"))
+		if err != nil {
+			return err
+		}
+
+		err = r.Put([]byte("0000000001"),
+			[]byte("{\"id\":\"1\",\"project_id\":\"1\",\"admin\": true}"))
+
+		return err
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = migration_2_8_91{migration{store.db}}.Apply()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var userData map[string]interface{}
+	err = store.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte("project__user_0000000001"))
+		str := string(b.Get([]byte("0000000001")))
+		return json.Unmarshal([]byte(str), &userData)
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if userData["role"].(string) != "owner" {
+		t.Fatal("invalid role")
+	}
+
+	if userData["admin"] != nil {
+		t.Fatal("admin field must be deleted")
+	}
+}
+
+func TestMigration_2_8_91_Apply2(t *testing.T) {
+	store := CreateTestStore()
+
+	err := store.db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("project"))
+		if err != nil {
+			return err
+		}
+
+		err = b.Put([]byte("0000000001"), []byte("{}"))
+
+		return err
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = migration_2_8_28{migration{store.db}}.Apply()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

db/bolt/option.go

@@ -0,0 +1,44 @@
+package bolt
+
+import (
+	"errors"
+	"github.com/ansible-semaphore/semaphore/db"
+)
+
+func (d *BoltDb) SetOption(key string, value string) error {
+
+	opt := db.Option{
+		Key:   key,
+		Value: value,
+	}
+
+	_, err := d.getOption(key)
+
+	if errors.Is(err, db.ErrNotFound) {
+		_, err = d.createObject(-1, db.OptionProps, opt)
+		return err
+	} else {
+		err = d.updateObject(-1, db.OptionProps, opt)
+	}
+
+	return err
+}
+
+func (d *BoltDb) getOption(key string) (value string, err error) {
+	var option db.Option
+	err = d.getObject(-1, db.OptionProps, strObjectID(key), &option)
+	value = option.Value
+	return
+}
+
+func (d *BoltDb) GetOption(key string) (value string, err error) {
+	var option db.Option
+	err = d.getObject(-1, db.OptionProps, strObjectID(key), &option)
+	value = option.Value
+
+	if errors.Is(err, db.ErrNotFound) {
+		err = nil
+	}
+
+	return
+}

db/bolt/option_test.go

@@ -0,0 +1,52 @@
+package bolt
+
+import (
+	"testing"
+)
+
+func TestGetOption(t *testing.T) {
+	store := CreateTestStore()
+
+	val, err := store.GetOption("unknown_option")
+
+	if err != nil && val != "" {
+		t.Fatal("Result must be empty string for non-existent option")
+	}
+}
+
+func TestGetSetOption(t *testing.T) {
+	store := CreateTestStore()
+
+	err := store.SetOption("age", "33")
+
+	if err != nil {
+		t.Fatal("Can not save option")
+	}
+
+	val, err := store.GetOption("age")
+
+	if err != nil {
+		t.Fatal("Can not get option")
+	}
+
+	if val != "33" {
+		t.Fatal("Invalid option value")
+	}
+
+	err = store.SetOption("age", "22")
+
+	if err != nil {
+		t.Fatal("Can not save option")
+	}
+
+	val, err = store.GetOption("age")
+
+	if err != nil {
+		t.Fatal("Can not get option")
+	}
+
+	if val != "22" {
+		t.Fatal("Invalid option value")
+	}
+
+}

db/bolt/project.go

@@ -17,6 +17,12 @@ func (d *BoltDb) CreateProject(project db.Project) (db.Project, error) {
 	return newProject.(db.Project), nil
 }
 
+func (d *BoltDb) GetAllProjects() (projects []db.Project, err error) {
+	err = d.getObjects(0, db.ProjectProps, db.RetrieveQueryParams{}, nil, &projects)
+
+	return
+}
+
 func (d *BoltDb) GetProjects(userID int) (projects []db.Project, err error) {
 	projects = make([]db.Project, 0)
 

db/bolt/project_test.go

@@ -34,7 +34,7 @@ func TestGetProjects(t *testing.T) {
 	_, err = store.CreateProjectUser(db.ProjectUser{
 		ProjectID: proj1.ID,
 		UserID:    usr.ID,
-		Admin:     true,
+		Role:      db.ProjectOwner,
 	})
 
 	if err != nil {

db/bolt/runner.go

@@ -0,0 +1,48 @@
+package bolt
+
+import (
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/ansible-semaphore/semaphore/util"
+)
+
+func (d *BoltDb) GetRunner(projectID int, runnerID int) (runner db.Runner, err error) {
+	return
+}
+
+func (d *BoltDb) GetRunners(projectID int) (runners []db.Runner, err error) {
+	return
+}
+
+func (d *BoltDb) DeleteRunner(projectID int, runnerID int) (err error) {
+	return
+}
+
+func (d *BoltDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
+	err = d.getObject(0, db.GlobalRunnerProps, intObjectID(runnerID), &runner)
+
+	return
+}
+
+func (d *BoltDb) GetGlobalRunners() (runners []db.Runner, err error) {
+	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, nil, &runners)
+	return
+}
+
+func (d *BoltDb) DeleteGlobalRunner(runnerID int) (err error) {
+	return
+}
+
+func (d *BoltDb) UpdateRunner(runner db.Runner) (err error) {
+	return
+}
+
+func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
+	runner.Token = util.RandString(12)
+
+	res, err := d.createObject(0, db.GlobalRunnerProps, runner)
+	if err != nil {
+		return
+	}
+	newRunner = res.(db.Runner)
+	return
+}

db/bolt/user.go

@@ -141,7 +141,7 @@ func (d *BoltDb) GetProjectUser(projectID, userID int) (user db.ProjectUser, err
 	return
 }
 
-func (d *BoltDb) GetProjectUsers(projectID int, params db.RetrieveQueryParams) (users []db.User, err error) {
+func (d *BoltDb) GetProjectUsers(projectID int, params db.RetrieveQueryParams) (users []db.UserWithProjectRole, err error) {
 	var projectUsers []db.ProjectUser
 	err = d.getObjects(projectID, db.ProjectUserProps, params, nil, &projectUsers)
 	if err != nil {
@@ -153,8 +153,11 @@ func (d *BoltDb) GetProjectUsers(projectID int, params db.RetrieveQueryParams) (
 		if err != nil {
 			return
 		}
-		usr.Admin = projUser.Admin
+		var usrWithRole = db.UserWithProjectRole{
-		users = append(users, usr)
+			User: usr,
+			Role: projUser.Role,
+		}
+		users = append(users, usrWithRole)
 	}
 	return
 }
@@ -167,7 +170,7 @@ func (d *BoltDb) DeleteProjectUser(projectID, userID int) error {
 	return d.deleteObject(projectID, db.ProjectUserProps, intObjectID(userID), nil)
 }
 
-//GetUser retrieves a user from the database by ID
+// GetUser retrieves a user from the database by ID
 func (d *BoltDb) GetUser(userID int) (user db.User, err error) {
 	err = d.getObject(0, db.UserProps, intObjectID(userID), &user)
 	return

db/bolt/user_test.go

@@ -34,14 +34,14 @@ func TestBoltDb_UpdateProjectUser(t *testing.T) {
 	projUser, err := store.CreateProjectUser(db.ProjectUser{
 		ProjectID: proj1.ID,
 		UserID:    usr.ID,
-		Admin:     true,
+		Role:      db.ProjectOwner,
 	})
 
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	projUser.Admin = true
+	projUser.Role = db.ProjectOwner
 	err = store.UpdateProjectUser(projUser)
 
 	if err != nil {

db/sql/SqlDb.go

@@ -2,19 +2,20 @@ package sql
 
 import (
 	"database/sql"
+	"embed"
 	"fmt"
-	log "github.com/Sirupsen/logrus"
-	"github.com/ansible-semaphore/semaphore/db"
-	"github.com/ansible-semaphore/semaphore/util"
-	"github.com/go-gorp/gorp/v3"
-	_ "github.com/go-sql-driver/mysql" // imports mysql driver
-	"github.com/gobuffalo/packr"
-	_ "github.com/lib/pq"
-	"github.com/masterminds/squirrel"
 	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
+
+	"github.com/Masterminds/squirrel"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/ansible-semaphore/semaphore/util"
+	"github.com/go-gorp/gorp/v3"
+	_ "github.com/go-sql-driver/mysql" // imports mysql driver
+	_ "github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
 )
 
 type SqlDb struct {
@@ -28,7 +29,9 @@ create table ` + "`migrations`" + ` (
 	` + "`notes`" + ` text null
 );
 `
-var dbAssets = packr.NewBox("./migrations")
+
+//go:embed migrations/*.sql
+var dbAssets embed.FS
 
 func containsStr(arr []string, str string) bool {
 	for _, a := range arr {
@@ -150,7 +153,7 @@ func connect() (*sql.DB, error) {
 		return nil, err
 	}
 
-	dialect := cfg.Dialect.String()
+	dialect := cfg.Dialect
 	return sql.Open(dialect, connectionString)
 }
 
@@ -169,7 +172,7 @@ func createDb() error {
 		return err
 	}
 
-	conn, err := sql.Open(cfg.Dialect.String(), connectionString)
+	conn, err := sql.Open(cfg.Dialect, connectionString)
 	if err != nil {
 		return err
 	}
@@ -211,8 +214,11 @@ func (d *SqlDb) getObject(projectID int, props db.ObjectProps, objectID int, obj
 
 func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
 	q := squirrel.Select("*").
-		From(props.TableName+" pe").
+		From(props.TableName + " pe")
-		Where("pe.project_id=?", projectID)
+
+	if !props.IsGlobal {
+		q = q.Where("pe.project_id=?", projectID)
+	}
 
 	orderDirection := "ASC"
 	if params.SortInverted {
@@ -228,6 +234,14 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.Retrie
 		q = q.OrderBy("pe." + orderColumn + " " + orderDirection)
 	}
 
+	if params.Count > 0 {
+		q = q.Limit(uint64(params.Count))
+	}
+
+	if params.Offset > 0 {
+		q = q.Offset(uint64(params.Offset))
+	}
+
 	query, args, err := q.ToSql()
 
 	if err != nil {
@@ -239,12 +253,23 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.Retrie
 	return
 }
 
+func (d *SqlDb) getProjectObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
+	return d.getObjects(projectID, props, params, objects)
+}
+
 func (d *SqlDb) deleteObject(projectID int, props db.ObjectProps, objectID int) error {
-	return validateMutationResult(
+	if props.IsGlobal {
-		d.exec(
+		return validateMutationResult(
-			"delete from "+props.TableName+" where project_id=? and id=?",
+			d.exec(
-			projectID,
+				"delete from "+props.TableName+" where id=?",
-			objectID))
+				objectID))
+	} else {
+		return validateMutationResult(
+			d.exec(
+				"delete from "+props.TableName+" where project_id=? and id=?",
+				projectID,
+				objectID))
+	}
 }
 
 func (d *SqlDb) Close(token string) {
@@ -452,3 +477,300 @@ func (d *SqlDb) IsInitialized() (bool, error) {
 	_, err := d.sql.SelectInt(d.PrepareQuery("select count(1) from migrations"))
 	return err == nil, nil
 }
+
+func (d *SqlDb) getObjectByReferrer(referrerID int, referringObjectProps db.ObjectProps, props db.ObjectProps, objectID int, object interface{}) (err error) {
+	query, args, err := squirrel.Select("*").
+		From(props.TableName).
+		Where("id=?", objectID).
+		Where(referringObjectProps.ReferringColumnSuffix+"=?", referrerID).
+		ToSql()
+
+	if err != nil {
+		return
+	}
+
+	err = d.selectOne(object, query, args...)
+
+	if err == sql.ErrNoRows {
+		err = db.ErrNotFound
+	}
+
+	return
+}
+
+func (d *SqlDb) getObjectsByReferrer(referrerID int, referringObjectProps db.ObjectProps, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
+	var referringColumn = referringObjectProps.ReferringColumnSuffix
+
+	q := squirrel.Select("*").
+		From(props.TableName + " pe")
+
+	if props.IsGlobal {
+		q = q.Where("pe." + referringColumn + " is null")
+	} else {
+		q = q.Where("pe."+referringColumn+"=?", referrerID)
+	}
+
+	orderDirection := "ASC"
+	if params.SortInverted {
+		orderDirection = "DESC"
+	}
+
+	orderColumn := props.DefaultSortingColumn
+	if containsStr(props.SortableColumns, params.SortBy) {
+		orderColumn = params.SortBy
+	}
+
+	if orderColumn != "" {
+		q = q.OrderBy("pe." + orderColumn + " " + orderDirection)
+	}
+
+	query, args, err := q.ToSql()
+
+	if err != nil {
+		return
+	}
+
+	_, err = d.selectAll(objects, query, args...)
+
+	return
+}
+
+func (d *SqlDb) deleteByReferrer(referrerID int, referringObjectProps db.ObjectProps, props db.ObjectProps, objectID int) error {
+	var referringColumn = referringObjectProps.ReferringColumnSuffix
+
+	return validateMutationResult(
+		d.exec(
+			"delete from "+props.TableName+" where "+referringColumn+"=? and id=?",
+			referrerID,
+			objectID))
+}
+
+func (d *SqlDb) deleteObjectByReferencedID(referencedID int, referencedProps db.ObjectProps, props db.ObjectProps, objectID int) error {
+	field := referencedProps.ReferringColumnSuffix
+
+	return validateMutationResult(
+		d.exec("delete from "+props.TableName+" t where t."+field+"=? and t.id=?", referencedID, objectID))
+}
+
+/**
+  GENERIC IMPLEMENTATION
+  **/
+
+func InsertTemplateFromType(typeInstance interface{}) (string, []interface{}) {
+	val := reflect.Indirect(reflect.ValueOf(typeInstance))
+	typeFieldSize := val.Type().NumField()
+
+	fields := ""
+	values := ""
+	args := make([]interface{}, 0)
+
+	if typeFieldSize > 1 {
+		fields += "("
+		values += "("
+	}
+
+	for i := 0; i < typeFieldSize; i++ {
+		if val.Type().Field(i).Name == "ID" {
+			continue
+		}
+		fields += val.Type().Field(i).Tag.Get("db")
+		values += "?"
+		args = append(args, val.Field(i))
+		if i != (typeFieldSize - 1) {
+			fields += ", "
+			values += ", "
+		}
+	}
+
+	if typeFieldSize > 1 {
+		fields += ")"
+		values += ")"
+	}
+
+	return fields + " values " + values, args
+}
+
+func AddParams(params db.RetrieveQueryParams, q *squirrel.SelectBuilder, props db.ObjectProps) {
+	orderDirection := "ASC"
+	if params.SortInverted {
+		orderDirection = "DESC"
+	}
+
+	orderColumn := props.DefaultSortingColumn
+	if containsStr(props.SortableColumns, params.SortBy) {
+		orderColumn = params.SortBy
+	}
+
+	if orderColumn != "" {
+		q.OrderBy("t." + orderColumn + " " + orderDirection)
+	}
+}
+
+func (d *SqlDb) GetObject(props db.ObjectProps, ID int) (object interface{}, err error) {
+	query, args, err := squirrel.Select("t.*").
+		From(props.TableName + " as t").
+		Where(squirrel.Eq{"t.id": ID}).
+		OrderBy("t.id").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+	err = d.selectOne(&object, query, args...)
+
+	return
+}
+
+func (d *SqlDb) CreateObject(props db.ObjectProps, object interface{}) (newObject interface{}, err error) {
+	//err = newObject.Validate()
+
+	if err != nil {
+		return
+	}
+
+	template, args := InsertTemplateFromType(newObject)
+	insertID, err := d.insert(
+		"id",
+		"insert into "+props.TableName+" "+template, args...)
+
+	if err != nil {
+		return
+	}
+
+	newObject = object
+
+	v := reflect.ValueOf(newObject)
+	field := v.FieldByName("ID")
+	field.SetInt(int64(insertID))
+
+	return
+}
+
+func (d *SqlDb) GetObjectsByForeignKeyQuery(props db.ObjectProps, foreignID int, foreignProps db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
+	q := squirrel.Select("*").
+		From(props.TableName+" as t").
+		Where(foreignProps.ReferringColumnSuffix+"=?", foreignID)
+
+	AddParams(params, &q, props)
+
+	query, args, err := q.
+		OrderBy("t.id").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+	err = d.selectOne(&objects, query, args...)
+
+	return
+}
+
+func (d *SqlDb) GetAllObjectsByForeignKey(props db.ObjectProps, foreignID int, foreignProps db.ObjectProps) (objects interface{}, err error) {
+	query, args, err := squirrel.Select("*").
+		From(props.TableName+" as t").
+		Where(foreignProps.ReferringColumnSuffix+"=?", foreignID).
+		OrderBy("t.id").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+
+	results, errQuery := d.selectAll(&objects, query, args...)
+
+	return results, errQuery
+}
+
+func (d *SqlDb) GetAllObjects(props db.ObjectProps) (objects interface{}, err error) {
+	query, args, err := squirrel.Select("*").
+		From(props.TableName + " as t").
+		OrderBy("t.id").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+	var results []interface{}
+	results, err = d.selectAll(&objects, query, args...)
+
+	return results, err
+
+}
+
+// Retrieve the Matchers & Values referncing `id' from WebhookExtractor
+// --
+// Examples:
+// referrerCollection := db.ObjectReferrers{}
+//
+//	d.GetReferencesForForeignKey(db.ProjectProps, id, map[string]db.ObjectProps{
+//	  'Templates': db.TemplateProps,
+//	  'Inventories': db.InventoryProps,
+//	  'Repositories': db.RepositoryProps
+//	}, &referrerCollection)
+//
+// //
+//
+// referrerCollection := db.WebhookExtractorReferrers{}
+//
+//	d.GetReferencesForForeignKey(db.WebhookProps, id, map[string]db.ObjectProps{
+//	  "Matchers": db.WebhookMatcherProps,
+//	  "Values": db.WebhookExtractValueProps
+//	}, &referrerCollection)
+func (d *SqlDb) GetReferencesForForeignKey(objectProps db.ObjectProps, objectID int, referrerMapping map[string]db.ObjectProps, referrerCollection *interface{}) (err error) {
+
+	for key, value := range referrerMapping {
+		//v := reflect.ValueOf(referrerCollection)
+		referrers, errRef := d.GetObjectReferences(objectProps, value, objectID)
+
+		if errRef != nil {
+			return errRef
+		}
+		reflect.ValueOf(referrerCollection).FieldByName(key).Set(reflect.ValueOf(referrers))
+	}
+
+	return
+}
+
+// Find Object Referrers for objectID based on referring column taken from referringObjectProps
+// Example:
+// GetObjectReferences(db.WebhookMatchers, db.WebhookExtractorProps, integrationID)
+func (d *SqlDb) GetObjectReferences(objectProps db.ObjectProps, referringObjectProps db.ObjectProps, objectID int) (referringObjs []db.ObjectReferrer, err error) {
+	referringObjs = make([]db.ObjectReferrer, 0)
+
+	fields, err := objectProps.GetReferringFieldsFrom(objectProps.Type)
+
+	cond := ""
+	vals := []interface{}{}
+
+	for _, f := range fields {
+		if cond != "" {
+			cond += " or "
+		}
+
+		cond += f + " = ?"
+
+		vals = append(vals, objectID)
+	}
+
+	if cond == "" {
+		return
+	}
+
+	referringObjects := reflect.New(reflect.SliceOf(referringObjectProps.Type))
+	_, err = d.selectAll(
+		referringObjects.Interface(),
+		"select id, name from "+referringObjectProps.TableName+" where "+objectProps.ReferringColumnSuffix+" = ? and "+cond,
+		vals...)
+
+	if err != nil {
+		return
+	}
+
+	for i := 0; i < referringObjects.Elem().Len(); i++ {
+		id := int(referringObjects.Elem().Index(i).FieldByName("ID").Int())
+		name := referringObjects.Elem().Index(i).FieldByName("Name").String()
+		referringObjs = append(referringObjs, db.ObjectReferrer{ID: id, Name: name})
+	}
+
+	return
+}

db/sql/SqlDb_test.go

@@ -11,4 +11,4 @@ func TestValidatePort(t *testing.T) {
 	if q != "select * from \"test\" where id = $1, email = $2" {
 		t.Error("invalid postgres query")
 	}
-}
+}

db/sql/access_key.go

@@ -2,16 +2,12 @@ package sql
 
 import (
 	"database/sql"
+	"errors"
 	"github.com/ansible-semaphore/semaphore/db"
 )
 
 func (d *SqlDb) GetAccessKey(projectID int, accessKeyID int) (key db.AccessKey, err error) {
 	err = d.getObject(projectID, db.AccessKeyProps, accessKeyID, &key)
-
-	if err != nil {
-		return
-	}
-
 	return
 }
 
@@ -21,7 +17,7 @@ func (d *SqlDb) GetAccessKeyRefs(projectID int, keyID int) (db.ObjectReferrers,
 
 func (d *SqlDb) GetAccessKeys(projectID int, params db.RetrieveQueryParams) ([]db.AccessKey, error) {
 	var keys []db.AccessKey
-	err := d.getObjects(projectID, db.AccessKeyProps, params, &keys)
+	err := d.getProjectObjects(projectID, db.AccessKeyProps, params, &keys)
 	return keys, err
 }
 
@@ -87,3 +83,43 @@ func (d *SqlDb) CreateAccessKey(key db.AccessKey) (newKey db.AccessKey, err erro
 func (d *SqlDb) DeleteAccessKey(projectID int, accessKeyID int) error {
 	return d.deleteObject(projectID, db.AccessKeyProps, accessKeyID)
 }
+
+const RekeyBatchSize = 100
+
+func (d *SqlDb) RekeyAccessKeys(oldKey string) (err error) {
+
+	var globalProps = db.AccessKeyProps
+	globalProps.IsGlobal = true
+
+	for i := 0; ; i++ {
+
+		var keys []db.AccessKey
+		err = d.getObjects(-1, globalProps, db.RetrieveQueryParams{Count: RekeyBatchSize, Offset: i * RekeyBatchSize}, &keys)
+
+		if err != nil {
+			return
+		}
+
+		if len(keys) == 0 {
+			break
+		}
+
+		for _, key := range keys {
+
+			err = key.DeserializeSecret2(oldKey)
+
+			if err != nil {
+				return err
+			}
+
+			key.OverrideSecret = true
+			err = d.UpdateAccessKey(key)
+
+			if err != nil && !errors.Is(err, db.ErrNotFound) {
+				return err
+			}
+		}
+	}
+
+	return
+}

db/sql/environment.go

@@ -4,10 +4,9 @@ import (
 	"github.com/ansible-semaphore/semaphore/db"
 )
 
-func (d *SqlDb) GetEnvironment(projectID int, environmentID int) (db.Environment, error) {
+func (d *SqlDb) GetEnvironment(projectID int, environmentID int) (environment db.Environment, err error) {
-	var environment db.Environment
+	err = d.getObject(projectID, db.EnvironmentProps, environmentID, &environment)
-	err := d.getObject(projectID, db.EnvironmentProps, environmentID, &environment)
+	return
-	return environment, err
 }
 
 func (d *SqlDb) GetEnvironmentRefs(projectID int, environmentID int) (db.ObjectReferrers, error) {
@@ -16,7 +15,7 @@ func (d *SqlDb) GetEnvironmentRefs(projectID int, environmentID int) (db.ObjectR
 
 func (d *SqlDb) GetEnvironments(projectID int, params db.RetrieveQueryParams) ([]db.Environment, error) {
 	var environment []db.Environment
-	err := d.getObjects(projectID, db.EnvironmentProps, params, &environment)
+	err := d.getProjectObjects(projectID, db.EnvironmentProps, params, &environment)
 	return environment, err
 }
 
@@ -28,10 +27,11 @@ func (d *SqlDb) UpdateEnvironment(env db.Environment) error {
 	}
 
 	_, err = d.exec(
-		"update project__environment set name=?, json=?, env=? where id=?",
+		"update project__environment set name=?, json=?, env=?, password=? where id=?",
 		env.Name,
 		env.JSON,
 		env.ENV,
+		env.Password,
 		env.ID)
 	return err
 }

db/sql/event.go

@@ -1,8 +1,8 @@
 package sql
 
 import (
+	"github.com/Masterminds/squirrel"
 	"github.com/ansible-semaphore/semaphore/db"
-	"github.com/masterminds/squirrel"
 	"time"
 )
 

db/sql/integration.go

@@ -0,0 +1,300 @@
+package sql
+
+import (
+	"github.com/Masterminds/squirrel"
+	"github.com/ansible-semaphore/semaphore/db"
+)
+
+func (d *SqlDb) CreateIntegration(integration db.Integration) (newIntegration db.Integration, err error) {
+	err = integration.Validate()
+
+	if err != nil {
+		return
+	}
+
+	insertID, err := d.insert(
+		"id",
+		"insert into project__integration "+
+			"(project_id, name, template_id, auth_method, auth_secret_id, auth_header, searchable) values "+
+			"(?, ?, ?, ?, ?, ?, ?)",
+		integration.ProjectID,
+		integration.Name,
+		integration.TemplateID,
+		integration.AuthMethod,
+		integration.AuthSecretID,
+		integration.AuthHeader,
+		integration.Searchable)
+
+	if err != nil {
+		return
+	}
+
+	newIntegration = integration
+	newIntegration.ID = insertID
+
+	return
+}
+
+func (d *SqlDb) GetIntegrations(projectID int, params db.RetrieveQueryParams) (integrations []db.Integration, err error) {
+	err = d.getProjectObjects(projectID, db.IntegrationProps, params, &integrations)
+	return integrations, err
+}
+
+func (d *SqlDb) GetAllIntegrations() (integrations []db.Integration, err error) {
+	var integrationObjects interface{}
+	integrationObjects, err = d.GetAllObjects(db.IntegrationProps)
+	integrations = integrationObjects.([]db.Integration)
+	return
+}
+
+func (d *SqlDb) GetIntegration(projectID int, integrationID int) (integration db.Integration, err error) {
+	err = d.getObject(projectID, db.IntegrationProps, integrationID, &integration)
+	return
+}
+
+func (d *SqlDb) GetIntegrationRefs(projectID int, integrationID int) (referrers db.IntegrationReferrers, err error) {
+	//var extractorReferrer []db.ObjectReferrer
+	//extractorReferrer, err = d.GetObjectReferences(db.IntegrationProps, db.IntegrationExtractorProps, integrationID)
+	//referrers = db.IntegrationReferrers{
+	//	IntegrationExtractors: extractorReferrer,
+	//}
+	return
+}
+
+func (d *SqlDb) DeleteIntegration(projectID int, integrationID int) error {
+	//extractors, err := d.GetIntegrationExtractors(0, db.RetrieveQueryParams{}, integrationID)
+	//
+	//if err != nil {
+	//	return err
+	//}
+	//
+	//for extractor := range extractors {
+	//	d.DeleteIntegrationExtractor(0, extractors[extractor].ID, integrationID)
+	//}
+	return d.deleteObject(projectID, db.IntegrationProps, integrationID)
+}
+
+func (d *SqlDb) UpdateIntegration(integration db.Integration) error {
+	err := integration.Validate()
+
+	if err != nil {
+		return err
+	}
+
+	_, err = d.exec(
+		"update project__integration set `name`=?, template_id=?, auth_method=?, auth_secret_id=?, auth_header=?, searchable=? where `id`=?",
+		integration.Name,
+		integration.TemplateID,
+		integration.AuthMethod,
+		integration.AuthSecretID,
+		integration.AuthHeader,
+		integration.Searchable,
+		integration.ID)
+
+	return err
+}
+
+func (d *SqlDb) CreateIntegrationExtractValue(projectId int, value db.IntegrationExtractValue) (newValue db.IntegrationExtractValue, err error) {
+	err = value.Validate()
+
+	if err != nil {
+		return
+	}
+
+	insertID, err := d.insert("id",
+		"insert into project__integration_extract_value "+
+			"(value_source, body_data_type, `key`, `variable`, `name`, integration_id) values "+
+			"(?, ?, ?, ?, ?, ?)",
+		value.ValueSource,
+		value.BodyDataType,
+		value.Key,
+		value.Variable,
+		value.Name,
+		value.IntegrationID)
+
+	if err != nil {
+		return
+	}
+
+	newValue = value
+	newValue.ID = insertID
+
+	return
+}
+
+func (d *SqlDb) GetIntegrationExtractValues(projectID int, params db.RetrieveQueryParams, integrationID int) ([]db.IntegrationExtractValue, error) {
+	var values []db.IntegrationExtractValue
+	err := d.getObjectsByReferrer(integrationID, db.IntegrationProps, db.IntegrationExtractValueProps, params, &values)
+	return values, err
+}
+
+func (d *SqlDb) GetAllIntegrationExtractValues() (values []db.IntegrationExtractValue, err error) {
+	var valueObjects interface{}
+	valueObjects, err = d.GetAllObjects(db.IntegrationExtractValueProps)
+	values = valueObjects.([]db.IntegrationExtractValue)
+	return
+}
+
+func (d *SqlDb) GetIntegrationExtractValue(projectID int, valueID int, integrationID int) (value db.IntegrationExtractValue, err error) {
+	query, args, err := squirrel.Select("v.*").
+		From("project__integration_extract_value as v").
+		Where(squirrel.Eq{"id": valueID}).
+		OrderBy("v.id").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+
+	err = d.selectOne(&value, query, args...)
+
+	return value, err
+}
+
+func (d *SqlDb) GetIntegrationExtractValueRefs(projectID int, valueID int, integrationID int) (refs db.IntegrationExtractorChildReferrers, err error) {
+	refs.Integrations, err = d.GetObjectReferences(db.IntegrationProps, db.IntegrationExtractValueProps, integrationID)
+	return
+}
+
+func (d *SqlDb) DeleteIntegrationExtractValue(projectID int, valueID int, integrationID int) error {
+	return d.deleteObjectByReferencedID(integrationID, db.IntegrationProps, db.IntegrationExtractValueProps, valueID)
+}
+
+func (d *SqlDb) UpdateIntegrationExtractValue(projectID int, integrationExtractValue db.IntegrationExtractValue) error {
+	err := integrationExtractValue.Validate()
+
+	if err != nil {
+		return err
+	}
+
+	_, err = d.exec(
+		"update project__integration_extract_value set value_source=?, body_data_type=?, `key`=?, `variable`=?, `name`=? where `id`=?",
+		integrationExtractValue.ValueSource,
+		integrationExtractValue.BodyDataType,
+		integrationExtractValue.Key,
+		integrationExtractValue.Variable,
+		integrationExtractValue.Name,
+		integrationExtractValue.ID)
+
+	return err
+}
+
+func (d *SqlDb) CreateIntegrationMatcher(projectID int, matcher db.IntegrationMatcher) (newMatcher db.IntegrationMatcher, err error) {
+	err = matcher.Validate()
+
+	if err != nil {
+		return
+	}
+
+	insertID, err := d.insert(
+		"id",
+		"insert into project__integration_matcher "+
+			"(match_type, `method`, body_data_type, `key`, `value`, integration_id, `name`) values "+
+			"(?, ?, ?, ?, ?, ?, ?)",
+		matcher.MatchType,
+		matcher.Method,
+		matcher.BodyDataType,
+		matcher.Key,
+		matcher.Value,
+		matcher.IntegrationID,
+		matcher.Name)
+
+	if err != nil {
+		return
+	}
+
+	newMatcher = matcher
+	newMatcher.ID = insertID
+
+	return
+}
+
+func (d *SqlDb) GetIntegrationMatchers(projectID int, params db.RetrieveQueryParams, integrationID int) (matchers []db.IntegrationMatcher, err error) {
+	query, args, err := squirrel.Select("m.*").
+		From("project__integration_matcher as m").
+		Where(squirrel.Eq{"integration_id": integrationID}).
+		OrderBy("m.id").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+
+	_, err = d.selectAll(&matchers, query, args...)
+
+	return
+}
+
+func (d *SqlDb) GetAllIntegrationMatchers() (matchers []db.IntegrationMatcher, err error) {
+	var matcherObjects interface{}
+	matcherObjects, err = d.GetAllObjects(db.IntegrationMatcherProps)
+	matchers = matcherObjects.([]db.IntegrationMatcher)
+
+	return
+}
+
+func (d *SqlDb) GetIntegrationMatcher(projectID int, matcherID int, integrationID int) (matcher db.IntegrationMatcher, err error) {
+	query, args, err := squirrel.Select("m.*").
+		From("project__integration_matcher as m").
+		Where(squirrel.Eq{"id": matcherID}).
+		OrderBy("m.id").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+
+	err = d.selectOne(&matcher, query, args...)
+
+	return matcher, err
+}
+
+func (d *SqlDb) GetIntegrationMatcherRefs(projectID int, matcherID int, integrationID int) (refs db.IntegrationExtractorChildReferrers, err error) {
+	refs.Integrations, err = d.GetObjectReferences(db.IntegrationProps, db.IntegrationMatcherProps, matcherID)
+
+	return
+}
+
+func (d *SqlDb) DeleteIntegrationMatcher(projectID int, matcherID int, integrationID int) error {
+	return d.deleteObjectByReferencedID(integrationID, db.IntegrationProps, db.IntegrationMatcherProps, matcherID)
+}
+
+func (d *SqlDb) UpdateIntegrationMatcher(projectID int, integrationMatcher db.IntegrationMatcher) error {
+	err := integrationMatcher.Validate()
+
+	if err != nil {
+		return err
+	}
+
+	_, err = d.exec(
+		"update project__integration_matcher set match_type=?, `method`=?, body_data_type=?, `key`=?, `value`=?, `name`=? where `id`=?",
+		integrationMatcher.MatchType,
+		integrationMatcher.Method,
+		integrationMatcher.BodyDataType,
+		integrationMatcher.Key,
+		integrationMatcher.Value,
+		integrationMatcher.Name,
+		integrationMatcher.ID)
+
+	return err
+}
+
+func (d *SqlDb) CreateIntegrationAlias(alias db.IntegrationAlias) (res db.IntegrationAlias, err error) {
+	return
+}
+
+func (d *SqlDb) GetIntegrationAlias(projectID int, integrationID *int) (res db.IntegrationAlias, err error) {
+	return
+}
+
+func (d *SqlDb) GetIntegrationAliasByAlias(alias string) (res db.IntegrationAlias, err error) {
+	return
+}
+
+func (d *SqlDb) UpdateIntegrationAlias(alias db.IntegrationAlias) error {
+	return nil
+}
+
+func (d *SqlDb) DeleteIntegrationAlias(projectID int, integrationID *int) error {
+	return nil
+}

db/sql/inventory.go

@@ -14,7 +14,7 @@ func (d *SqlDb) GetInventory(projectID int, inventoryID int) (inventory db.Inven
 
 func (d *SqlDb) GetInventories(projectID int, params db.RetrieveQueryParams) ([]db.Inventory, error) {
 	var inventories []db.Inventory
-	err := d.getObjects(projectID, db.InventoryProps, params, &inventories)
+	err := d.getProjectObjects(projectID, db.InventoryProps, params, &inventories)
 	return inventories, err
 }
 
@@ -28,12 +28,13 @@ func (d *SqlDb) DeleteInventory(projectID int, inventoryID int) error {
 
 func (d *SqlDb) UpdateInventory(inventory db.Inventory) error {
 	_, err := d.exec(
-		"update project__inventory set name=?, type=?, ssh_key_id=?, inventory=?, become_key_id=? where id=?",
+		"update project__inventory set name=?, type=?, ssh_key_id=?, inventory=?, become_key_id=?, holder_id=? where id=?",
 		inventory.Name,
 		inventory.Type,
 		inventory.SSHKeyID,
 		inventory.Inventory,
 		inventory.BecomeKeyID,
+		inventory.HolderID,
 		inventory.ID)
 
 	return err
@@ -42,13 +43,15 @@ func (d *SqlDb) UpdateInventory(inventory db.Inventory) error {
 func (d *SqlDb) CreateInventory(inventory db.Inventory) (newInventory db.Inventory, err error) {
 	insertID, err := d.insert(
 		"id",
-		"insert into project__inventory (project_id, name, type, ssh_key_id, inventory, become_key_id) values (?, ?, ?, ?, ?, ?)",
+		"insert into project__inventory (project_id, name, type, ssh_key_id, inventory, become_key_id, holder_id) values "+
+			"(?, ?, ?, ?, ?, ?, ?)",
 		inventory.ProjectID,
 		inventory.Name,
 		inventory.Type,
 		inventory.SSHKeyID,
 		inventory.Inventory,
-		inventory.BecomeKeyID)
+		inventory.BecomeKeyID,
+		inventory.HolderID)
 
 	if err != nil {
 		return

db/sql/migration.go

@@ -2,12 +2,14 @@ package sql
 
 import (
 	"fmt"
-	log "github.com/Sirupsen/logrus"
-	"github.com/ansible-semaphore/semaphore/db"
 	"github.com/go-gorp/gorp/v3"
+	"path"
 	"regexp"
 	"strings"
 	"time"
+
+	"github.com/ansible-semaphore/semaphore/db"
+	log "github.com/sirupsen/logrus"
 )
 
 var (
@@ -32,14 +34,14 @@ func getVersionErrPath(version db.Migration) string {
 	return version.HumanoidVersion() + ".err.sql"
 }
 
-// getVersionSQL takes a path to an SQL file and returns it from packr as
+// getVersionSQL takes a path to an SQL file and returns it from embed.FS
 // a slice of strings separated by newlines
-func getVersionSQL(path string) (queries []string) {
+func getVersionSQL(name string) (queries []string) {
-	sql, err := dbAssets.MustString(path)
+	sql, err := dbAssets.ReadFile(path.Join("migrations", name))
 	if err != nil {
 		panic(err)
 	}
-	queries = strings.Split(strings.ReplaceAll(sql, ";\r\n", ";\n"), ";\n")
+	queries = strings.Split(strings.ReplaceAll(string(sql), ";\r\n", ";\n"), ";\n")
 	for i := range queries {
 		queries[i] = strings.Trim(queries[i], "\r\n\t ")
 	}
@@ -186,7 +188,7 @@ func (d *SqlDb) ApplyMigration(migration db.Migration) error {
 
 // TryRollbackMigration attempts to rollback the database to an earlier version if a rollback exists
 func (d *SqlDb) TryRollbackMigration(version db.Migration) {
-	data := dbAssets.Bytes(getVersionErrPath(version))
+	data, _ := dbAssets.ReadFile(getVersionErrPath(version))
 	if len(data) == 0 {
 		fmt.Println("Rollback SQL does not exist.")
 		fmt.Println()

db/sql/migrations/v2.8.91.sql

@@ -0,0 +1,5 @@
+ALTER TABLE project__user ADD `role` varchar(50) NOT NULL DEFAULT 'manager';
+
+UPDATE project__user SET `role` = 'owner' WHERE `admin`;
+
+ALTER TABLE project__user DROP COLUMN `admin`;

db/sql/migrations/v2.9.46.sql

@@ -0,0 +1 @@
+ALTER TABLE project__template ADD `app` varchar(50) NOT NULL DEFAULT '';

db/sql/migrations/v2.9.6.sql

@@ -0,0 +1,10 @@
+create table runner
+(
+    id                  integer primary key autoincrement,
+    project_id          int,
+    token               varchar(255) not null,
+    webhook             varchar(1000) not null default '',
+    max_parallel_tasks  int not null default 0,
+
+    foreign key (`project_id`) references project(`id`) on delete cascade
+);

db/sql/migrations/v2.9.60.sql

@@ -0,0 +1,46 @@
+create table project__integration (
+  `id` integer primary key autoincrement,
+  `name` varchar(255) not null,
+  `project_id` int not null,
+  `template_id` int not null,
+  `auth_method` varchar(15) not null default 'none',
+  `auth_secret_id` int,
+  `auth_header` varchar(255),
+
+  foreign key (`project_id`) references project(`id`) on delete cascade,
+  foreign key (`template_id`) references project__template(`id`) on delete cascade,
+  foreign key (`auth_secret_id`) references access_key(`id`) on delete set null
+);
+
+create table project__integration_extractor (
+  `id` integer primary key autoincrement,
+  `name` varchar(255) not null,
+  `integration_id` int not null,
+
+  foreign key (`integration_id`) references project__integration(`id`) on delete cascade
+);
+
+create table project__integration_extract_value (
+  `id` integer primary key autoincrement,
+  `name` varchar(255) not null,
+  `extractor_id` int not null,
+  `value_source` varchar(255) not null,
+  `body_data_type` varchar(255) null,
+  `key` varchar(255) null,
+  `variable` varchar(255) null,
+
+  foreign key (`extractor_id`) references project__integration_extractor(`id`) on delete cascade
+);
+
+create table project__integration_matcher (
+  `id` integer primary key autoincrement,
+  `name` varchar(255) not null,
+  `extractor_id` int not null,
+  `match_type` varchar(255) null,
+  `method` varchar(255) null,
+  `body_data_type` varchar(255) null,
+  `key` varchar(510) null,
+  `value` varchar(510) null,
+
+  foreign key (`extractor_id`) references project__integration_extractor(`id`) on delete cascade
+);

db/sql/migrations/v2.9.61.sql

@@ -0,0 +1,57 @@
+drop table project__integration_matcher;
+drop table project__integration_extract_value;
+drop table project__integration_extractor;
+drop table project__integration;
+
+create table project__integration (
+  `id` integer primary key autoincrement,
+  `name` varchar(255) not null,
+  `project_id` int not null,
+  `template_id` int not null,
+  `auth_method` varchar(15) not null default 'none',
+  `auth_secret_id` int,
+  `auth_header` varchar(255),
+  `searchable` bool not null default false,
+
+  foreign key (`project_id`) references project(`id`) on delete cascade,
+  foreign key (`template_id`) references project__template(`id`) on delete cascade,
+  foreign key (`auth_secret_id`) references access_key(`id`) on delete set null
+);
+
+create table project__integration_extract_value (
+  `id` integer primary key autoincrement,
+  `name` varchar(255) not null,
+  `integration_id` int not null,
+  `value_source` varchar(255) not null,
+  `body_data_type` varchar(255) null,
+  `key` varchar(255) null,
+  `variable` varchar(255) null,
+
+  foreign key (`integration_id`) references project__integration(`id`) on delete cascade
+);
+
+create table project__integration_matcher (
+  `id` integer primary key autoincrement,
+  `name` varchar(255) not null,
+  `integration_id` int not null,
+  `match_type` varchar(255) null,
+  `method` varchar(255) null,
+  `body_data_type` varchar(255) null,
+  `key` varchar(510) null,
+  `value` varchar(510) null,
+
+  foreign key (`integration_id`) references project__integration(`id`) on delete cascade
+);
+
+create table project__integration_alias (
+  `id` integer primary key autoincrement,
+  `alias` varchar(50) not null,
+  `project_id` int not null,
+  `integration_id` int,
+
+  foreign key (`project_id`) references project(`id`) on delete cascade,
+  foreign key (`integration_id`) references project__integration(`id`) on delete cascade,
+
+  unique (`alias`),
+  unique (`project_id`, `integration_id`)
+);

db/sql/migrations/v2.9.62.sql

@@ -0,0 +1,10 @@
+alter table project add `type` varchar(20) default '';
+
+alter table task add `inventory_id` int null references project__inventory(`id`) on delete set null;
+
+alter table project__inventory add `holder_id` int null references project__template(`id`) on delete set null;
+
+create table `option` (
+    `key` varchar(255) primary key not null,
+    `value` varchar(255) not null
+);

db/sql/option.go

@@ -0,0 +1,58 @@
+package sql
+
+import (
+	"database/sql"
+	"errors"
+	"github.com/Masterminds/squirrel"
+	"github.com/ansible-semaphore/semaphore/db"
+)
+
+func (d *SqlDb) SetOption(key string, value string) error {
+	_, err := d.getOption(key)
+
+	if errors.Is(err, db.ErrNotFound) {
+		_, err = d.exec("update option set value=? where key=?", key)
+	} else {
+		_, err = d.insert(
+			"key",
+			"insert into option (key, value) values (?, ?)",
+			key, value)
+	}
+
+	return err
+}
+
+func (d *SqlDb) getOption(key string) (value string, err error) {
+	q := squirrel.Select("*").
+		From(db.OptionProps.TableName).
+		Where("key=?", key)
+
+	query, args, err := q.ToSql()
+
+	if err != nil {
+		return
+	}
+
+	var opt db.Option
+
+	err = d.selectOne(&opt, query, args...)
+
+	if errors.Is(err, sql.ErrNoRows) {
+		err = db.ErrNotFound
+	}
+
+	value = opt.Value
+
+	return
+}
+
+func (d *SqlDb) GetOption(key string) (value string, err error) {
+
+	value, err = d.getOption(key)
+
+	if errors.Is(err, db.ErrNotFound) {
+		err = nil
+	}
+
+	return
+}

db/sql/project.go

@@ -1,8 +1,8 @@
 package sql
 
 import (
+	"github.com/Masterminds/squirrel"
 	"github.com/ansible-semaphore/semaphore/db"
-	"github.com/masterminds/squirrel"
 	"time"
 )
 
@@ -11,8 +11,8 @@ func (d *SqlDb) CreateProject(project db.Project) (newProject db.Project, err er
 
 	insertId, err := d.insert(
 		"id",
-		"insert into project(name, created) values (?, ?)",
+		"insert into project(name, created, type) values (?, ?, ?)",
-		project.Name, project.Created)
+		project.Name, project.Created, project.Type)
 
 	if err != nil {
 		return
@@ -23,6 +23,21 @@ func (d *SqlDb) CreateProject(project db.Project) (newProject db.Project, err er
 	return
 }
 
+func (d *SqlDb) GetAllProjects() (projects []db.Project, err error) {
+	query, args, err := squirrel.Select("p.*").
+		From("project as p").
+		OrderBy("p.name").
+		ToSql()
+
+	if err != nil {
+		return
+	}
+
+	_, err = d.selectAll(&projects, query, args...)
+
+	return
+}
+
 func (d *SqlDb) GetProjects(userID int) (projects []db.Project, err error) {
 	query, args, err := squirrel.Select("p.*").
 		From("project as p").
@@ -56,6 +71,14 @@ func (d *SqlDb) GetProject(projectID int) (project db.Project, err error) {
 }
 
 func (d *SqlDb) DeleteProject(projectID int) error {
+
+	//tpls, err := d.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{})
+	//
+	//if err != nil {
+	//	return err
+	//}
+	// TODO: sort projects
+
 	tx, err := d.sql.Begin()
 
 	if err != nil {
@@ -75,7 +98,7 @@ func (d *SqlDb) DeleteProject(projectID int) error {
 		_, err = tx.Exec(d.PrepareQuery(statement), projectID)
 
 		if err != nil {
-			err = tx.Rollback()
+			_ = tx.Rollback()
 			return err
 		}
 	}

db/sql/repository.go

@@ -2,7 +2,7 @@ package sql
 
 import (
 	"github.com/ansible-semaphore/semaphore/db"
-	"github.com/masterminds/squirrel"
+	"github.com/Masterminds/squirrel"
 )
 
 func (d *SqlDb) GetRepository(projectID int, repositoryID int) (db.Repository, error) {

db/sql/runner.go

@@ -0,0 +1,65 @@
+package sql
+
+import (
+	"encoding/base64"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/securecookie"
+)
+
+func (d *SqlDb) GetRunner(projectID int, runnerID int) (runner db.Runner, err error) {
+	return
+}
+
+func (d *SqlDb) GetRunners(projectID int) (runners []db.Runner, err error) {
+	return
+}
+
+func (d *SqlDb) DeleteRunner(projectID int, runnerID int) (err error) {
+	return
+}
+
+func (d *SqlDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
+	err = d.getObject(0, db.GlobalRunnerProps, runnerID, &runner)
+	return
+}
+
+func (d *SqlDb) GetGlobalRunners() (runners []db.Runner, err error) {
+	err = d.getProjectObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, &runners)
+	return
+}
+
+func (d *SqlDb) DeleteGlobalRunner(runnerID int) (err error) {
+	err = d.deleteObject(0, db.GlobalRunnerProps, runnerID)
+	return
+}
+
+func (d *SqlDb) UpdateRunner(runner db.Runner) (err error) {
+	_, err = d.exec(
+		"update runner set webhook=?, max_parallel_tasks=? where id=?",
+		runner.Webhook,
+		runner.MaxParallelTasks,
+		runner.ID)
+
+	return
+}
+
+func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
+	token := base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
+
+	insertID, err := d.insert(
+		"id",
+		"insert into runner (project_id, token, webhook, max_parallel_tasks) values (?, ?, ?, ?)",
+		runner.ProjectID,
+		token,
+		runner.Webhook,
+		runner.MaxParallelTasks)
+
+	if err != nil {
+		return
+	}
+
+	newRunner = runner
+	newRunner.ID = insertID
+	newRunner.Token = token
+	return
+}

db/sql/task.go

@@ -3,7 +3,7 @@ package sql
 import (
 	"database/sql"
 	"github.com/ansible-semaphore/semaphore/db"
-	"github.com/masterminds/squirrel"
+	"github.com/Masterminds/squirrel"
 )
 
 func (d *SqlDb) CreateTask(task db.Task) (db.Task, error) {

db/sql/template.go

@@ -2,8 +2,9 @@ package sql
 
 import (
 	"database/sql"
+
 	"github.com/ansible-semaphore/semaphore/db"
-	"github.com/masterminds/squirrel"
+	"github.com/Masterminds/squirrel"
 )
 
 func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, err error) {
@@ -17,8 +18,8 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e
 		"id",
 		"insert into project__template (project_id, inventory_id, repository_id, environment_id, "+
 			"name, playbook, arguments, allow_override_args_in_task, description, vault_key_id, `type`, start_version,"+
-			"build_template_id, view_id, autorun, survey_vars, suppress_success_alerts)"+
+			"build_template_id, view_id, autorun, survey_vars, suppress_success_alerts, app)"+
-			"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+			"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
 		template.ProjectID,
 		template.InventoryID,
 		template.RepositoryID,
@@ -35,7 +36,8 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e
 		template.ViewID,
 		template.Autorun,
 		db.ObjectToJSON(template.SurveyVars),
-		template.SuppressSuccessAlerts)
+		template.SuppressSuccessAlerts,
+		template.App)
 
 	if err != nil {
 		return
@@ -76,7 +78,8 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
 		"view_id=?, "+
 		"autorun=?, "+
 		"survey_vars=?, "+
-		"suppress_success_alerts=? "+
+		"suppress_success_alerts=?, "+
+		"app=? "+
 		"where id=? and project_id=?",
 		template.InventoryID,
 		template.RepositoryID,
@@ -94,6 +97,7 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
 		template.Autorun,
 		db.ObjectToJSON(template.SurveyVars),
 		template.SuppressSuccessAlerts,
+		template.App,
 		template.ID,
 		template.ProjectID,
 	)
@@ -111,7 +115,12 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
 		"pt.arguments",
 		"pt.allow_override_args_in_task",
 		"pt.vault_key_id",
+		"pt.build_template_id",
+		"pt.start_version",
 		"pt.view_id",
+		"pt.`app`",
+		"pt.survey_vars",
+		"pt.start_version",
 		"pt.`type`").
 		From("project__template pt")
 

db/sql/user.go

@@ -3,7 +3,7 @@ package sql
 import (
 	"database/sql"
 	"github.com/ansible-semaphore/semaphore/db"
-	"github.com/masterminds/squirrel"
+	"github.com/Masterminds/squirrel"
 	"golang.org/x/crypto/bcrypt"
 	"time"
 )
@@ -104,10 +104,10 @@ func (d *SqlDb) SetUserPassword(userID int, password string) error {
 
 func (d *SqlDb) CreateProjectUser(projectUser db.ProjectUser) (newProjectUser db.ProjectUser, err error) {
 	_, err = d.exec(
-		"insert into project__user (project_id, user_id, `admin`) values (?, ?, ?)",
+		"insert into project__user (project_id, user_id, `role`) values (?, ?, ?)",
 		projectUser.ProjectID,
 		projectUser.UserID,
-		projectUser.Admin)
+		projectUser.Role)
 
 	if err != nil {
 		return
@@ -132,8 +132,9 @@ func (d *SqlDb) GetProjectUser(projectID, userID int) (db.ProjectUser, error) {
 	return user, err
 }
 
-func (d *SqlDb) GetProjectUsers(projectID int, params db.RetrieveQueryParams) (users []db.User, err error) {
+func (d *SqlDb) GetProjectUsers(projectID int, params db.RetrieveQueryParams) (users []db.UserWithProjectRole, err error) {
-	q := squirrel.Select("u.*").Column("pu.admin").
+	q := squirrel.Select("u.*").
+		Column("pu.role").
 		From("project__user as pu").
 		LeftJoin("`user` as u on pu.user_id=u.id").
 		Where("pu.project_id=?", projectID)
@@ -146,7 +147,7 @@ func (d *SqlDb) GetProjectUsers(projectID int, params db.RetrieveQueryParams) (u
 	switch params.SortBy {
 	case "name", "username", "email":
 		q = q.OrderBy("u." + params.SortBy + " " + sortDirection)
-	case "admin":
+	case "role":
 		q = q.OrderBy("pu." + params.SortBy + " " + sortDirection)
 	default:
 		q = q.OrderBy("u.name " + sortDirection)
@@ -165,8 +166,8 @@ func (d *SqlDb) GetProjectUsers(projectID int, params db.RetrieveQueryParams) (u
 
 func (d *SqlDb) UpdateProjectUser(projectUser db.ProjectUser) error {
 	_, err := d.exec(
-		"update `project__user` set admin=? where user_id=? and project_id = ?",
+		"update `project__user` set role=? where user_id=? and project_id = ?",
-		projectUser.Admin,
+		projectUser.Role,
 		projectUser.UserID,
 		projectUser.ProjectID)
 
@@ -178,7 +179,7 @@ func (d *SqlDb) DeleteProjectUser(projectID, userID int) error {
 	return err
 }
 
-//GetUser retrieves a user from the database by ID
+// GetUser retrieves a user from the database by ID
 func (d *SqlDb) GetUser(userID int) (db.User, error) {
 	var user db.User
 

db/sql/view.go

@@ -8,7 +8,7 @@ func (d *SqlDb) GetView(projectID int, viewID int) (view db.View, err error) {
 }
 
 func (d *SqlDb) GetViews(projectID int) (views []db.View, err error) {
-	err = d.getObjects(projectID, db.ViewProps, db.RetrieveQueryParams{}, &views)
+	err = d.getProjectObjects(projectID, db.ViewProps, db.RetrieveQueryParams{}, &views)
 	return
 }
 

db_lib/AnsibleApp.go

@@ -0,0 +1,157 @@
+package db_lib
+
+import (
+	"crypto/md5"
+	"fmt"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/ansible-semaphore/semaphore/lib"
+	"io"
+	"os"
+	"path"
+)
+
+func getMD5Hash(filepath string) (string, error) {
+	file, err := os.Open(filepath)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}
+
+func hasRequirementsChanges(requirementsFilePath string, requirementsHashFilePath string) bool {
+	oldFileMD5HashBytes, err := os.ReadFile(requirementsHashFilePath)
+	if err != nil {
+		return true
+	}
+
+	newFileMD5Hash, err := getMD5Hash(requirementsFilePath)
+	if err != nil {
+		return true
+	}
+
+	return string(oldFileMD5HashBytes) != newFileMD5Hash
+}
+
+func writeMD5Hash(requirementsFile string, requirementsHashFile string) error {
+	newFileMD5Hash, err := getMD5Hash(requirementsFile)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(requirementsHashFile, []byte(newFileMD5Hash), 0644)
+}
+
+type AnsibleApp struct {
+	Logger     lib.Logger
+	Playbook   *AnsiblePlaybook
+	Template   db.Template
+	Repository db.Repository
+}
+
+func (t *AnsibleApp) SetLogger(logger lib.Logger) lib.Logger {
+	t.Logger = logger
+	return logger
+}
+
+func (t *AnsibleApp) Run(args []string, environmentVars *[]string, cb func(*os.Process)) error {
+	return t.Playbook.RunPlaybook(args, environmentVars, cb)
+}
+
+func (t *AnsibleApp) Log(msg string) {
+	t.Logger.Log(msg)
+}
+
+func (t *AnsibleApp) InstallRequirements() error {
+	if err := t.installCollectionsRequirements(); err != nil {
+		return err
+	}
+	if err := t.installRolesRequirements(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *AnsibleApp) getRepoPath() string {
+	repo := GitRepository{
+		Logger:     t.Logger,
+		TemplateID: t.Template.ID,
+		Repository: t.Repository,
+		Client:     CreateDefaultGitClient(),
+	}
+
+	return repo.GetFullPath()
+}
+
+func (t *AnsibleApp) installRolesRequirements() error {
+	requirementsFilePath := fmt.Sprintf("%s/roles/requirements.yml", t.getRepoPath())
+	requirementsHashFilePath := fmt.Sprintf("%s.md5", requirementsFilePath)
+
+	if _, err := os.Stat(requirementsFilePath); err != nil {
+		t.Log("No roles/requirements.yml file found. Skip galaxy install process.\n")
+		return nil
+	}
+
+	if hasRequirementsChanges(requirementsFilePath, requirementsHashFilePath) {
+		if err := t.runGalaxy([]string{
+			"role",
+			"install",
+			"-r",
+			requirementsFilePath,
+			"--force",
+		}); err != nil {
+			return err
+		}
+		if err := writeMD5Hash(requirementsFilePath, requirementsHashFilePath); err != nil {
+			return err
+		}
+	} else {
+		t.Log("roles/requirements.yml has no changes. Skip galaxy install process.\n")
+	}
+
+	return nil
+}
+
+func (t *AnsibleApp) GetPlaybookDir() string {
+	playbookPath := path.Join(t.getRepoPath(), t.Template.Playbook)
+
+	return path.Dir(playbookPath)
+}
+
+func (t *AnsibleApp) installCollectionsRequirements() error {
+	requirementsFilePath := path.Join(t.GetPlaybookDir(), "collections", "requirements.yml")
+	requirementsHashFilePath := fmt.Sprintf("%s.md5", requirementsFilePath)
+
+	if _, err := os.Stat(requirementsFilePath); err != nil {
+		t.Log("No collections/requirements.yml file found. Skip galaxy install process.\n")
+		return nil
+	}
+
+	if hasRequirementsChanges(requirementsFilePath, requirementsHashFilePath) {
+		if err := t.runGalaxy([]string{
+			"collection",
+			"install",
+			"-r",
+			requirementsFilePath,
+			"--force",
+		}); err != nil {
+			return err
+		}
+		if err := writeMD5Hash(requirementsFilePath, requirementsHashFilePath); err != nil {
+			return err
+		}
+	} else {
+		t.Log("collections/requirements.yml has no changes. Skip galaxy install process.\n")
+	}
+
+	return nil
+}
+
+func (t *AnsibleApp) runGalaxy(args []string) error {
+	return t.Playbook.RunGalaxy(args)
+}