Runners (#1477)

* feat(runners): add migartion

* feat(runners): sql implementation

db/Migration.go

@@ -60,6 +60,7 @@ func GetMigrations() []Migration {
 		{Version: "2.8.57"},
 		{Version: "2.8.58"},
 		{Version: "2.8.91"},
+		{Version: "2.9.6"},
 	}
 }
 

db/Store.go

@@ -314,7 +314,7 @@ var ViewProps = ObjectProps{
 	DefaultSortingColumn: "position",
 }
 
-var RunnerProps = ObjectProps{
+var GlobalRunnerProps = ObjectProps{
 	TableName:         "runner",
 	Type:              reflect.TypeOf(Runner{}),
 	PrimaryColumnName: "id",

db/bolt/runner.go

@@ -18,13 +18,13 @@ func (d *BoltDb) DeleteRunner(projectID int, runnerID int) (err error) {
 }
 
 func (d *BoltDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
-	err = d.getObject(0, db.RunnerProps, intObjectID(runnerID), &runner)
+	err = d.getObject(0, db.GlobalRunnerProps, intObjectID(runnerID), &runner)
 
 	return
 }
 
 func (d *BoltDb) GetGlobalRunners() (runners []db.Runner, err error) {
-	err = d.getObjects(0, db.RunnerProps, db.RetrieveQueryParams{}, nil, &runners)
+	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, nil, &runners)
 	return
 }
 
@@ -39,7 +39,7 @@ func (d *BoltDb) UpdateRunner(runner db.Runner) (err error) {
 func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
 	runner.Token = util.RandString(12)
 
-	res, err := d.createObject(0, db.RunnerProps, runner)
+	res, err := d.createObject(0, db.GlobalRunnerProps, runner)
 	if err != nil {
 		return
 	}

db/sql/SqlDb.go

@@ -211,8 +211,13 @@ func (d *SqlDb) getObject(projectID int, props db.ObjectProps, objectID int, obj
 
 func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
 	q := squirrel.Select("*").
-		From(props.TableName+" pe").
+		From(props.TableName + " pe")
-		Where("pe.project_id=?", projectID)
+
+	if props.IsGlobal {
+		q = q.Where("pe.project_id is null")
+	} else {
+		q = q.Where("pe.project_id=?", projectID)
+	}
 
 	orderDirection := "ASC"
 	if params.SortInverted {
@@ -240,11 +245,18 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.Retrie
 }
 
 func (d *SqlDb) deleteObject(projectID int, props db.ObjectProps, objectID int) error {
-	return validateMutationResult(
+	if props.IsGlobal {
-		d.exec(
+		return validateMutationResult(
-			"delete from "+props.TableName+" where project_id=? and id=?",
+			d.exec(
-			projectID,
+				"delete from "+props.TableName+" where id=?",
-			objectID))
+				objectID))
+	} else {
+		return validateMutationResult(
+			d.exec(
+				"delete from "+props.TableName+" where project_id=? and id=?",
+				projectID,
+				objectID))
+	}
 }
 
 func (d *SqlDb) Close(token string) {

db/sql/access_key.go

@@ -8,10 +8,6 @@ import (
 func (d *SqlDb) GetAccessKey(projectID int, accessKeyID int) (key db.AccessKey, err error) {
 	err = d.getObject(projectID, db.AccessKeyProps, accessKeyID, &key)
 
-	if err != nil {
-		return
-	}
-
 	return
 }
 

db/sql/migrations/v2.9.6.sql

@@ -0,0 +1,10 @@
+create table runner
+(
+    id                  integer primary key autoincrement,
+    project_id          int,
+    token               varchar(255) not null,
+    webhook             varchar(1000) not null default '',
+    max_parallel_tasks  int not null default 0,
+
+    foreign key (`project_id`) references project(`id`) on delete cascade
+);

db/sql/runner.go

@@ -1,6 +1,10 @@
 package sql
 
-import "github.com/ansible-semaphore/semaphore/db"
+import (
+	"encoding/base64"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/gorilla/securecookie"
+)
 
 func (d *SqlDb) GetRunner(projectID int, runnerID int) (runner db.Runner, err error) {
 	return
@@ -15,21 +19,47 @@ func (d *SqlDb) DeleteRunner(projectID int, runnerID int) (err error) {
 }
 
 func (d *SqlDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
+	err = d.getObject(0, db.GlobalRunnerProps, runnerID, &runner)
 	return
 }
 
 func (d *SqlDb) GetGlobalRunners() (runners []db.Runner, err error) {
+	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, &runners)
 	return
 }
 
 func (d *SqlDb) DeleteGlobalRunner(runnerID int) (err error) {
+	err = d.deleteObject(0, db.GlobalRunnerProps, runnerID)
 	return
 }
 
 func (d *SqlDb) UpdateRunner(runner db.Runner) (err error) {
+	_, err = d.exec(
+		"update runner set webhook=?, max_parallel_tasks=? where id=?",
+		runner.Webhook,
+		runner.MaxParallelTasks,
+		runner.ID)
+
 	return
 }
 
 func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
+	token := base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
+
+	insertID, err := d.insert(
+		"id",
+		"insert into runner (project_id, token, webhook, max_parallel_tasks) values (?, ?, ?, ?)",
+		runner.ProjectID,
+		token,
+		runner.Webhook,
+		runner.MaxParallelTasks)
+
+	if err != nil {
+		return
+	}
+
+	newRunner = runner
+	newRunner.ID = insertID
+	newRunner.Token = token
 	return
 }

deployment/docker/common/runner-wrapper

@@ -0,0 +1,164 @@
+#!/bin/sh
+
+set -e
+
+echoerr() { printf "%s\n" "$*" >&2; }
+
+file_env() {
+        local var=""
+        local fileVar=""
+        eval var="\$${1}"
+        eval fileVar="\$${1}_FILE"
+        local def="${2:-}"
+        if [ -n "${var:-}" ] && [ -n "${fileVar:-}" ]; then
+                echo >&2 "error: both ${1} and ${1}_FILE are set (but are exclusive)"
+                exit 1
+        fi
+        local val="$def"
+        if [ -n "${var:-}" ]; then
+                val="${var}"
+        elif [ -n "${fileVar:-}" ]; then
+                val="$(cat "${fileVar}")"
+        fi
+        export "${1}"="$val"
+        unset "${1}_FILE"
+}
+
+SEMAPHORE_CONFIG_PATH="${SEMAPHORE_CONFIG_PATH:-/etc/semaphore}"
+SEMAPHORE_TMP_PATH="${SEMAPHORE_TMP_PATH:-/tmp/semaphore}"
+
+# Semaphore database env config
+SEMAPHORE_DB_DIALECT="${SEMAPHORE_DB_DIALECT:-mysql}"
+SEMAPHORE_DB_HOST="${SEMAPHORE_DB_HOST:-0.0.0.0}"
+SEMAPHORE_DB_PATH="${SEMAPHORE_DB_PATH:-/var/lib/semaphore}"
+SEMAPHORE_DB_PORT="${SEMAPHORE_DB_PORT:-}"
+SEMAPHORE_DB="${SEMAPHORE_DB:-semaphore}"
+file_env 'SEMAPHORE_DB_USER' 'semaphore'
+file_env 'SEMAPHORE_DB_PASS' 'semaphore'
+# Email alert env config
+SEMAPHORE_WEB_ROOT="${SEMAPHORE_WEB_ROOT:-}"
+# Semaphore Admin env config
+file_env 'SEMAPHORE_ADMIN' 'admin'
+SEMAPHORE_ADMIN_EMAIL="${SEMAPHORE_ADMIN_EMAIL:-admin@localhost}"
+SEMAPHORE_ADMIN_NAME="${SEMAPHORE_ADMIN_NAME:-Semaphore Admin}"
+file_env 'SEMAPHORE_ADMIN_PASSWORD' 'semaphorepassword'
+#Semaphore LDAP env config
+SEMAPHORE_LDAP_ACTIVATED="${SEMAPHORE_LDAP_ACTIVATED:-no}"
+SEMAPHORE_LDAP_HOST="${SEMAPHORE_LDAP_HOST:-}"
+SEMAPHORE_LDAP_PORT="${SEMAPHORE_LDAP_PORT:-}"
+SEMAPHORE_LDAP_NEEDTLS="${SEMAPHORE_LDAP_NEEDTLS:-no}"
+SEMAPHORE_LDAP_DN_BIND="${SEMAPHORE_LDAP_DN_BIND:-}"
+file_env 'SEMAPHORE_LDAP_PASSWORD'
+SEMAPHORE_LDAP_DN_SEARCH="${SEMAPHORE_LDAP_DN_SEARCH:-}"
+SEMAPHORE_LDAP_SEARCH_FILTER="${SEMAPHORE_LDAP_SEARCH_FILTER:-(uid=%s)}"
+SEMAPHORE_LDAP_MAPPING_DN="${SEMAPHORE_LDAP_MAPPING_DN:-dn}"
+SEMAPHORE_LDAP_MAPPING_USERNAME="${SEMAPHORE_LDAP_MAPPING_USERNAME:-uid}"
+SEMAPHORE_LDAP_MAPPING_FULLNAME="${SEMAPHORE_LDAP_MAPPING_FULLNAME:-cn}"
+SEMAPHORE_LDAP_MAPPING_EMAIL="${SEMAPHORE_LDAP_MAPPING_EMAIL:-mail}"
+
+file_env 'SEMAPHORE_ACCESS_KEY_ENCRYPTION'
+
+[ -d "${SEMAPHORE_TMP_PATH}" ] || mkdir -p "${SEMAPHORE_TMP_PATH}" || {
+    echo "Can't create Semaphore tmp path ${SEMAPHORE_TMP_PATH}."
+    exit 1
+}
+
+[ -d "${SEMAPHORE_CONFIG_PATH}" ] || mkdir -p "${SEMAPHORE_CONFIG_PATH}" || {
+    echo "Can't create Semaphore Config path ${SEMAPHORE_CONFIG_PATH}."
+    exit 1
+}
+
+[ -d "${SEMAPHORE_DB_PATH}" ] || mkdir -p "${SEMAPHORE_DB_PATH}" || {
+    echo "Can't create data path ${SEMAPHORE_DB_PATH}."
+    exit 1
+}
+
+# wait on db to be up
+
+if [ "${SEMAPHORE_DB_DIALECT}" != 'bolt' ]; then
+  echoerr "Attempting to connect to database ${SEMAPHORE_DB} on ${SEMAPHORE_DB_HOST}:${SEMAPHORE_DB_PORT} with user ${SEMAPHORE_DB_USER} ..."
+  TIMEOUT=30
+  while ! $(nc -z "$SEMAPHORE_DB_HOST" "$SEMAPHORE_DB_PORT") >/dev/null 2>&1; do
+      TIMEOUT=$(expr $TIMEOUT - 1)
+      if [ $TIMEOUT -eq 0 ]; then
+          echoerr "Could not connect to database server. Exiting."
+          exit 1
+      fi
+      echo -n "."
+      sleep 1
+  done
+fi
+
+if [ -n "${SEMAPHORE_DB_PORT}" ]; then
+  SEMAPHORE_DB_HOST="${SEMAPHORE_DB_HOST}:${SEMAPHORE_DB_PORT}"
+fi
+
+case ${SEMAPHORE_DB_DIALECT} in
+    mysql) SEMAPHORE_DB_DIALECT_ID=1;;
+    bolt) SEMAPHORE_DB_DIALECT_ID=2;;
+    postgres) SEMAPHORE_DB_DIALECT_ID=3;;
+    *)
+        echoerr "Unknown database dialect: ${SEMAPHORE_DB_DIALECT}"
+        exit 1
+        ;;
+esac
+
+# Create a config if it does not exist in the current config path
+if [ ! -f "${SEMAPHORE_CONFIG_PATH}/config.json" ]; then
+    echoerr "Generating ${SEMAPHORE_TMP_PATH}/config.stdin ..."
+    cat << EOF > "${SEMAPHORE_TMP_PATH}/config.stdin"
+${SEMAPHORE_DB_DIALECT_ID}
+EOF
+
+    if [ "${SEMAPHORE_DB_DIALECT}" = "bolt" ]; then
+      cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
+${SEMAPHORE_DB_PATH}/database.boltdb
+EOF
+    else
+      cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
+${SEMAPHORE_DB_HOST}
+${SEMAPHORE_DB_USER}
+${SEMAPHORE_DB_PASS}
+${SEMAPHORE_DB}
+EOF
+    fi
+
+      cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
+${SEMAPHORE_TMP_PATH}
+${SEMAPHORE_WEB_ROOT}
+no
+no
+no
+${SEMAPHORE_LDAP_ACTIVATED}
+EOF
+
+    if [ "${SEMAPHORE_LDAP_ACTIVATED}" = "yes" ]; then
+        cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
+${SEMAPHORE_LDAP_HOST}:${SEMAPHORE_LDAP_PORT}
+${SEMAPHORE_LDAP_NEEDTLS}
+${SEMAPHORE_LDAP_DN_BIND}
+${SEMAPHORE_LDAP_PASSWORD}
+${SEMAPHORE_LDAP_DN_SEARCH}
+${SEMAPHORE_LDAP_SEARCH_FILTER}
+${SEMAPHORE_LDAP_MAPPING_DN}
+${SEMAPHORE_LDAP_MAPPING_USERNAME}
+${SEMAPHORE_LDAP_MAPPING_FULLNAME}
+${SEMAPHORE_LDAP_MAPPING_EMAIL}
+EOF
+    fi;
+
+    cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
+${SEMAPHORE_CONFIG_PATH}
+${SEMAPHORE_ADMIN}
+${SEMAPHORE_ADMIN_EMAIL}
+${SEMAPHORE_ADMIN_NAME}
+${SEMAPHORE_ADMIN_PASSWORD}
+EOF
+
+    cat "${SEMAPHORE_TMP_PATH}/config.stdin"
+    $1 setup - < "${SEMAPHORE_TMP_PATH}/config.stdin"
+    echoerr "Run Semaphore with semaphore server --config ${SEMAPHORE_CONFIG_PATH}/config.json"
+fi
+
+# run our command
+exec "$@"

deployment/docker/dev/runner.Dockerfile

@@ -0,0 +1,40 @@
+FROM golang:1.19-alpine3.16
+
+ENV SEMAPHORE_VERSION="development" SEMAPHORE_ARCH="linux_amd64" \
+    SEMAPHORE_CONFIG_PATH="${SEMAPHORE_CONFIG_PATH:-/etc/semaphore}" \
+    APP_ROOT="/go/src/github.com/ansible-semaphore/semaphore/"
+
+# hadolint ignore=DL3013
+RUN apk add --no-cache gcc g++ sshpass git mysql-client python3 py3-pip py-openssl openssl ca-certificates curl curl-dev openssh-client-default tini nodejs npm bash rsync && \
+    apk --update add --virtual build-dependencies python3-dev libffi-dev openssl-dev build-base &&\
+    rm -rf /var/cache/apk/*
+
+RUN pip3 install --upgrade pip cffi &&\
+    apk del build-dependencies   && \
+    pip3 install ansible
+
+RUN adduser -D -u 1002 -g 0 semaphore && \
+    mkdir -p /go/src/github.com/ansible-semaphore/semaphore && \
+    mkdir -p /tmp/semaphore && \
+    mkdir -p /etc/semaphore && \
+    mkdir -p /var/lib/semaphore && \
+    chown -R semaphore:0 /go && \
+    chown -R semaphore:0 /tmp/semaphore && \
+    chown -R semaphore:0 /etc/semaphore && \
+    chown -R semaphore:0 /var/lib/semaphore && \
+    ssh-keygen -t rsa -q -f "/root/.ssh/id_rsa" -N ""       && \
+    ssh-keyscan -H github.com > /root/.ssh/known_hosts
+
+RUN cd $(go env GOPATH) && curl -sL https://taskfile.dev/install.sh | sh
+
+RUN git config --global --add safe.directory /go/src/github.com/ansible-semaphore/semaphore
+
+# Copy in app source
+WORKDIR ${APP_ROOT}
+COPY . ${APP_ROOT}
+RUN deployment/docker/dev/bin/install
+
+USER semaphore
+EXPOSE 3000
+ENTRYPOINT ["/usr/local/bin/runner-wrapper"]
+CMD ["./bin/semaphore", "runner", "--config", "/etc/semaphore/config.json"]

deployment/systemd/runner.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Semaphore Runner
+Requires=network.target
+
+[Service]
+EnvironmentFile=/etc/semaphore/env
+ExecStart=/usr/bin/semaphore runner --config ${SEMAPHORE_CONFIG}
+ExecReload=/bin/kill -HUP $MAINPID
+User=semaphore
+Group=semaphore
+Restart=always
+RestartSec=3s
+
+[Install]
+WantedBy=multi-user.target