Kiu/Semaphore
Kiu/Semaphore
1
0
Fork 0
mirror of https://github.com/semaphoreui/semaphore.git synced 2025-01-20 15:29:28 +01:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Runners (#1477)

Browse Source 
* feat(runners): add migartion

* feat(runners): sql implementation
This commit is contained in:
Denis Gukov 2023-09-11 01:02:10 +02:00 committed by GitHub
parent a8921e6e82
commit 6fa3e50198
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 284 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/Migration.go
View File

@ -60,6 +60,7 @@ func GetMigrations() []Migration {
{Version: "2.8.57"},
{Version: "2.8.58"},
{Version: "2.8.91"},
{Version: "2.9.6"},
}
}

2
db/Store.go
View File

@ -314,7 +314,7 @@ var ViewProps = ObjectProps{
DefaultSortingColumn: "position",
}
var RunnerProps = ObjectProps{
var GlobalRunnerProps = ObjectProps{
TableName: "runner",
Type: reflect.TypeOf(Runner{}),
PrimaryColumnName: "id",

6
db/bolt/runner.go
View File

@ -18,13 +18,13 @@ func (d *BoltDb) DeleteRunner(projectID int, runnerID int) (err error) {
}
func (d *BoltDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
err = d.getObject(0, db.RunnerProps, intObjectID(runnerID), &runner)
err = d.getObject(0, db.GlobalRunnerProps, intObjectID(runnerID), &runner)
return
}
func (d *BoltDb) GetGlobalRunners() (runners []db.Runner, err error) {
err = d.getObjects(0, db.RunnerProps, db.RetrieveQueryParams{}, nil, &runners)
err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, nil, &runners)
return
}
@ -39,7 +39,7 @@ func (d *BoltDb) UpdateRunner(runner db.Runner) (err error) {
func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
runner.Token = util.RandString(12)
res, err := d.createObject(0, db.RunnerProps, runner)
res, err := d.createObject(0, db.GlobalRunnerProps, runner)
if err != nil {
return
}

26
db/sql/SqlDb.go
View File

@ -211,8 +211,13 @@ func (d *SqlDb) getObject(projectID int, props db.ObjectProps, objectID int, obj
func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
q := squirrel.Select("*").
From(props.TableName+" pe").
Where("pe.project_id=?", projectID)
From(props.TableName + " pe")
if props.IsGlobal {
q = q.Where("pe.project_id is null")
} else {
q = q.Where("pe.project_id=?", projectID)
}
orderDirection := "ASC"
if params.SortInverted {
@ -240,11 +245,18 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.Retrie
}
func (d *SqlDb) deleteObject(projectID int, props db.ObjectProps, objectID int) error {
return validateMutationResult(
d.exec(
"delete from "+props.TableName+" where project_id=? and id=?",
projectID,
objectID))
if props.IsGlobal {
return validateMutationResult(
d.exec(
"delete from "+props.TableName+" where id=?",
objectID))
} else {
return validateMutationResult(
d.exec(
"delete from "+props.TableName+" where project_id=? and id=?",
projectID,
objectID))
}
}
func (d *SqlDb) Close(token string) {

4
db/sql/access_key.go
View File

@ -8,10 +8,6 @@ import (
func (d *SqlDb) GetAccessKey(projectID int, accessKeyID int) (key db.AccessKey, err error) {
err = d.getObject(projectID, db.AccessKeyProps, accessKeyID, &key)
if err != nil {
return
}
return
}

10
db/sql/migrations/v2.9.6.sql Normal file
View File

@ -0,0 +1,10 @@
create table runner
(
id integer primary key autoincrement,
project_id int,
token varchar(255) not null,
webhook varchar(1000) not null default '',
max_parallel_tasks int not null default 0,
foreign key (`project_id`) references project(`id`) on delete cascade
);

32
db/sql/runner.go
View File

@ -1,6 +1,10 @@
package sql
import "github.com/ansible-semaphore/semaphore/db"
import (
"encoding/base64"
"github.com/ansible-semaphore/semaphore/db"
"github.com/gorilla/securecookie"
)
func (d *SqlDb) GetRunner(projectID int, runnerID int) (runner db.Runner, err error) {
return
@ -15,21 +19,47 @@ func (d *SqlDb) DeleteRunner(projectID int, runnerID int) (err error) {
}
func (d *SqlDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
err = d.getObject(0, db.GlobalRunnerProps, runnerID, &runner)
return
}
func (d *SqlDb) GetGlobalRunners() (runners []db.Runner, err error) {
err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, &runners)
return
}
func (d *SqlDb) DeleteGlobalRunner(runnerID int) (err error) {
err = d.deleteObject(0, db.GlobalRunnerProps, runnerID)
return
}
func (d *SqlDb) UpdateRunner(runner db.Runner) (err error) {
_, err = d.exec(
"update runner set webhook=?, max_parallel_tasks=? where id=?",
runner.Webhook,
runner.MaxParallelTasks,
runner.ID)
return
}
func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
token := base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
insertID, err := d.insert(
"id",
"insert into runner (project_id, token, webhook, max_parallel_tasks) values (?, ?, ?, ?)",
runner.ProjectID,
token,
runner.Webhook,
runner.MaxParallelTasks)
if err != nil {
return
}
newRunner = runner
newRunner.ID = insertID
newRunner.Token = token
return
}

164
deployment/docker/common/runner-wrapper Executable file
View File

@ -0,0 +1,164 @@
#!/bin/sh
set -e
echoerr() { printf "%s\n" "$*" >&2; }
file_env() {
local var=""
local fileVar=""
eval var="\$${1}"
eval fileVar="\$${1}_FILE"
local def="${2:-}"
if [ -n "${var:-}" ] && [ -n "${fileVar:-}" ]; then
echo >&2 "error: both ${1} and ${1}_FILE are set (but are exclusive)"
exit 1
fi
local val="$def"
if [ -n "${var:-}" ]; then
val="${var}"
elif [ -n "${fileVar:-}" ]; then
val="$(cat "${fileVar}")"
fi
export "${1}"="$val"
unset "${1}_FILE"
}
SEMAPHORE_CONFIG_PATH="${SEMAPHORE_CONFIG_PATH:-/etc/semaphore}"
SEMAPHORE_TMP_PATH="${SEMAPHORE_TMP_PATH:-/tmp/semaphore}"
# Semaphore database env config
SEMAPHORE_DB_DIALECT="${SEMAPHORE_DB_DIALECT:-mysql}"
SEMAPHORE_DB_HOST="${SEMAPHORE_DB_HOST:-0.0.0.0}"
SEMAPHORE_DB_PATH="${SEMAPHORE_DB_PATH:-/var/lib/semaphore}"
SEMAPHORE_DB_PORT="${SEMAPHORE_DB_PORT:-}"
SEMAPHORE_DB="${SEMAPHORE_DB:-semaphore}"
file_env 'SEMAPHORE_DB_USER' 'semaphore'
file_env 'SEMAPHORE_DB_PASS' 'semaphore'
# Email alert env config
SEMAPHORE_WEB_ROOT="${SEMAPHORE_WEB_ROOT:-}"
# Semaphore Admin env config
file_env 'SEMAPHORE_ADMIN' 'admin'
SEMAPHORE_ADMIN_EMAIL="${SEMAPHORE_ADMIN_EMAIL:-admin@localhost}"
SEMAPHORE_ADMIN_NAME="${SEMAPHORE_ADMIN_NAME:-Semaphore Admin}"
file_env 'SEMAPHORE_ADMIN_PASSWORD' 'semaphorepassword'
#Semaphore LDAP env config
SEMAPHORE_LDAP_ACTIVATED="${SEMAPHORE_LDAP_ACTIVATED:-no}"
SEMAPHORE_LDAP_HOST="${SEMAPHORE_LDAP_HOST:-}"
SEMAPHORE_LDAP_PORT="${SEMAPHORE_LDAP_PORT:-}"
SEMAPHORE_LDAP_NEEDTLS="${SEMAPHORE_LDAP_NEEDTLS:-no}"
SEMAPHORE_LDAP_DN_BIND="${SEMAPHORE_LDAP_DN_BIND:-}"
file_env 'SEMAPHORE_LDAP_PASSWORD'
SEMAPHORE_LDAP_DN_SEARCH="${SEMAPHORE_LDAP_DN_SEARCH:-}"
SEMAPHORE_LDAP_SEARCH_FILTER="${SEMAPHORE_LDAP_SEARCH_FILTER:-(uid=%s)}"
SEMAPHORE_LDAP_MAPPING_DN="${SEMAPHORE_LDAP_MAPPING_DN:-dn}"
SEMAPHORE_LDAP_MAPPING_USERNAME="${SEMAPHORE_LDAP_MAPPING_USERNAME:-uid}"
SEMAPHORE_LDAP_MAPPING_FULLNAME="${SEMAPHORE_LDAP_MAPPING_FULLNAME:-cn}"
SEMAPHORE_LDAP_MAPPING_EMAIL="${SEMAPHORE_LDAP_MAPPING_EMAIL:-mail}"
file_env 'SEMAPHORE_ACCESS_KEY_ENCRYPTION'
[ -d "${SEMAPHORE_TMP_PATH}" ] || mkdir -p "${SEMAPHORE_TMP_PATH}" || {
echo "Can't create Semaphore tmp path ${SEMAPHORE_TMP_PATH}."
exit 1
}
[ -d "${SEMAPHORE_CONFIG_PATH}" ] || mkdir -p "${SEMAPHORE_CONFIG_PATH}" || {
echo "Can't create Semaphore Config path ${SEMAPHORE_CONFIG_PATH}."
exit 1
}
[ -d "${SEMAPHORE_DB_PATH}" ] || mkdir -p "${SEMAPHORE_DB_PATH}" || {
echo "Can't create data path ${SEMAPHORE_DB_PATH}."
exit 1
}
# wait on db to be up
if [ "${SEMAPHORE_DB_DIALECT}" != 'bolt' ]; then
echoerr "Attempting to connect to database ${SEMAPHORE_DB} on ${SEMAPHORE_DB_HOST}:${SEMAPHORE_DB_PORT} with user ${SEMAPHORE_DB_USER} ..."
TIMEOUT=30
while ! $(nc -z "$SEMAPHORE_DB_HOST" "$SEMAPHORE_DB_PORT") >/dev/null 2>&1; do
TIMEOUT=$(expr $TIMEOUT - 1)
if [ $TIMEOUT -eq 0 ]; then
echoerr "Could not connect to database server. Exiting."
exit 1
fi
echo -n "."
sleep 1
done
fi
if [ -n "${SEMAPHORE_DB_PORT}" ]; then
SEMAPHORE_DB_HOST="${SEMAPHORE_DB_HOST}:${SEMAPHORE_DB_PORT}"
fi
case ${SEMAPHORE_DB_DIALECT} in
mysql) SEMAPHORE_DB_DIALECT_ID=1;;
bolt) SEMAPHORE_DB_DIALECT_ID=2;;
postgres) SEMAPHORE_DB_DIALECT_ID=3;;
*)
echoerr "Unknown database dialect: ${SEMAPHORE_DB_DIALECT}"
exit 1
;;
esac
# Create a config if it does not exist in the current config path
if [ ! -f "${SEMAPHORE_CONFIG_PATH}/config.json" ]; then
echoerr "Generating ${SEMAPHORE_TMP_PATH}/config.stdin ..."
cat << EOF > "${SEMAPHORE_TMP_PATH}/config.stdin"
${SEMAPHORE_DB_DIALECT_ID}
EOF
if [ "${SEMAPHORE_DB_DIALECT}" = "bolt" ]; then
cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
${SEMAPHORE_DB_PATH}/database.boltdb
EOF
else
cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
${SEMAPHORE_DB_HOST}
${SEMAPHORE_DB_USER}
${SEMAPHORE_DB_PASS}
${SEMAPHORE_DB}
EOF
fi
cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
${SEMAPHORE_TMP_PATH}
${SEMAPHORE_WEB_ROOT}
no
no
no
${SEMAPHORE_LDAP_ACTIVATED}
EOF
if [ "${SEMAPHORE_LDAP_ACTIVATED}" = "yes" ]; then
cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
${SEMAPHORE_LDAP_HOST}:${SEMAPHORE_LDAP_PORT}
${SEMAPHORE_LDAP_NEEDTLS}
${SEMAPHORE_LDAP_DN_BIND}
${SEMAPHORE_LDAP_PASSWORD}
${SEMAPHORE_LDAP_DN_SEARCH}
${SEMAPHORE_LDAP_SEARCH_FILTER}
${SEMAPHORE_LDAP_MAPPING_DN}
${SEMAPHORE_LDAP_MAPPING_USERNAME}
${SEMAPHORE_LDAP_MAPPING_FULLNAME}
${SEMAPHORE_LDAP_MAPPING_EMAIL}
EOF
fi;
cat << EOF >> "${SEMAPHORE_TMP_PATH}/config.stdin"
${SEMAPHORE_CONFIG_PATH}
${SEMAPHORE_ADMIN}
${SEMAPHORE_ADMIN_EMAIL}
${SEMAPHORE_ADMIN_NAME}
${SEMAPHORE_ADMIN_PASSWORD}
EOF
cat "${SEMAPHORE_TMP_PATH}/config.stdin"
$1 setup - < "${SEMAPHORE_TMP_PATH}/config.stdin"
echoerr "Run Semaphore with semaphore server --config ${SEMAPHORE_CONFIG_PATH}/config.json"
fi
# run our command
exec "$@"

40
deployment/docker/dev/runner.Dockerfile Normal file
View File

@ -0,0 +1,40 @@
FROM golang:1.19-alpine3.16
ENV SEMAPHORE_VERSION="development" SEMAPHORE_ARCH="linux_amd64" \
SEMAPHORE_CONFIG_PATH="${SEMAPHORE_CONFIG_PATH:-/etc/semaphore}" \
APP_ROOT="/go/src/github.com/ansible-semaphore/semaphore/"
# hadolint ignore=DL3013
RUN apk add --no-cache gcc g++ sshpass git mysql-client python3 py3-pip py-openssl openssl ca-certificates curl curl-dev openssh-client-default tini nodejs npm bash rsync && \
apk --update add --virtual build-dependencies python3-dev libffi-dev openssl-dev build-base &&\
rm -rf /var/cache/apk/*
RUN pip3 install --upgrade pip cffi &&\
apk del build-dependencies && \
pip3 install ansible
RUN adduser -D -u 1002 -g 0 semaphore && \
mkdir -p /go/src/github.com/ansible-semaphore/semaphore && \
mkdir -p /tmp/semaphore && \
mkdir -p /etc/semaphore && \
mkdir -p /var/lib/semaphore && \
chown -R semaphore:0 /go && \
chown -R semaphore:0 /tmp/semaphore && \
chown -R semaphore:0 /etc/semaphore && \
chown -R semaphore:0 /var/lib/semaphore && \
ssh-keygen -t rsa -q -f "/root/.ssh/id_rsa" -N "" && \
ssh-keyscan -H github.com > /root/.ssh/known_hosts
RUN cd $(go env GOPATH) && curl -sL https://taskfile.dev/install.sh | sh
RUN git config --global --add safe.directory /go/src/github.com/ansible-semaphore/semaphore
# Copy in app source
WORKDIR ${APP_ROOT}
COPY . ${APP_ROOT}
RUN deployment/docker/dev/bin/install
USER semaphore
EXPOSE 3000
ENTRYPOINT ["/usr/local/bin/runner-wrapper"]
CMD ["./bin/semaphore", "runner", "--config", "/etc/semaphore/config.json"]

15
deployment/systemd/runner.service Normal file
View File

@ -0,0 +1,15 @@
[Unit]
Description=Semaphore Runner
Requires=network.target
[Service]
EnvironmentFile=/etc/semaphore/env
ExecStart=/usr/bin/semaphore runner --config ${SEMAPHORE_CONFIG}
ExecReload=/bin/kill -HUP $MAINPID
User=semaphore
Group=semaphore
Restart=always
RestartSec=3s
[Install]
WantedBy=multi-user.target