Merge pull request #2161 from semaphoreui/env-secrets

Envionment Secrets

api/projects/environment.go

@@ -10,6 +10,55 @@ import (
 	"github.com/gorilla/context"
 )
 
+func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
+	for _, secret := range env.Secrets {
+		var err error
+
+		var key db.AccessKey
+
+		switch secret.Operation {
+		case db.EnvironmentSecretCreate:
+			key, err = store.CreateAccessKey(db.AccessKey{
+				Name:          secret.Name,
+				String:        secret.Secret,
+				EnvironmentID: &env.ID,
+				ProjectID:     &env.ProjectID,
+				Type:          db.AccessKeyString,
+			})
+		case db.EnvironmentSecretDelete:
+			key, err = store.GetAccessKey(env.ProjectID, secret.ID)
+
+			if err != nil {
+				continue
+			}
+
+			if key.EnvironmentID == nil && *key.EnvironmentID == env.ID {
+				continue
+			}
+
+			err = store.DeleteAccessKey(env.ProjectID, secret.ID)
+		case db.EnvironmentSecretUpdate:
+			key, err = store.GetAccessKey(env.ProjectID, secret.ID)
+
+			if err != nil {
+				continue
+			}
+
+			if key.EnvironmentID == nil && *key.EnvironmentID == env.ID {
+				continue
+			}
+
+			err = store.UpdateAccessKey(db.AccessKey{
+				Name:   secret.Name,
+				String: secret.Secret,
+				Type:   db.AccessKeyString,
+			})
+		}
+	}
+
+	return nil
+}
+
 // EnvironmentMiddleware ensures an environment exists and loads it to the context
 func EnvironmentMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -27,6 +76,20 @@ func EnvironmentMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
+		keys, err := helpers.Store(r).GetEnvironmentSecrets(env.ProjectID, env.ID)
+
+		if err != nil {
+			helpers.WriteError(w, err)
+			return
+		}
+
+		for _, k := range keys {
+			env.Secrets = append(env.Secrets, db.EnvironmentSecret{
+				ID:   k.ID,
+				Name: k.Name,
+			})
+		}
+
 		context.Set(r, "environment", env)
 		next.ServeHTTP(w, r)
 	})
@@ -99,6 +162,11 @@ func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
 		Description: fmt.Sprintf("Environment %s updated", env.Name),
 	})
 
+	if err := updateEnvironmentSecrets(helpers.Store(r), env); err != nil {
+		helpers.WriteError(w, err)
+		return
+	}
+
 	w.WriteHeader(http.StatusNoContent)
 }
 
@@ -131,6 +199,11 @@ func AddEnvironment(w http.ResponseWriter, r *http.Request) {
 		Description: fmt.Sprintf("Environment %s created", newEnv.Name),
 	})
 
+	if err = updateEnvironmentSecrets(helpers.Store(r), newEnv); err != nil {
+		//helpers.WriteError(w, err)
+		//return
+	}
+
 	w.WriteHeader(http.StatusNoContent)
 }
 

db/AccessKey.go

@@ -21,6 +21,7 @@ const (
 	AccessKeySSH           AccessKeyType = "ssh"
 	AccessKeyNone          AccessKeyType = "none"
 	AccessKeyLoginPassword AccessKeyType = "login_password"
+	AccessKeyString        AccessKeyType = "string"
 )
 
 // AccessKey represents a key used to access a machine with ansible from semaphore
@@ -36,9 +37,12 @@ type AccessKey struct {
 	// You should use methods SerializeSecret to fill this field.
 	Secret *string `db:"secret" json:"-"`
 
+	String         string        `db:"-" json:"string"`
 	LoginPassword  LoginPassword `db:"-" json:"login_password"`
 	SshKey         SshKey        `db:"-" json:"ssh"`
 	OverrideSecret bool          `db:"-" json:"override_secret"`
+
+	EnvironmentID *int `db:"environment_id" json:"-"`
 }
 
 type LoginPassword struct {
@@ -167,6 +171,8 @@ func (key *AccessKey) SerializeSecret() error {
 	var err error
 
 	switch key.Type {
+	case AccessKeyString:
+		plaintext = []byte(key.String)
 	case AccessKeySSH:
 		plaintext, err = json.Marshal(key.SshKey)
 		if err != nil {
@@ -221,6 +227,8 @@ func (key *AccessKey) SerializeSecret() error {
 
 func (key *AccessKey) unmarshalAppropriateField(secret []byte) (err error) {
 	switch key.Type {
+	case AccessKeyString:
+		key.String = string(secret)
 	case AccessKeySSH:
 		sshKey := SshKey{}
 		err = json.Unmarshal(secret, &sshKey)

db/Environment.go

@@ -4,14 +4,30 @@ import (
 	"encoding/json"
 )
 
+type EnvironmentSecretOperation string
+
+const (
+	EnvironmentSecretCreate EnvironmentSecretOperation = "create"
+	EnvironmentSecretUpdate EnvironmentSecretOperation = "update"
+	EnvironmentSecretDelete EnvironmentSecretOperation = "delete"
+)
+
+type EnvironmentSecret struct {
+	ID        int                        `json:"id"`
+	Name      string                     `json:"name"`
+	Secret    string                     `json:"secret"`
+	Operation EnvironmentSecretOperation `json:"operation"`
+}
+
 // Environment is used to pass additional arguments, in json form to ansible
 type Environment struct {
-	ID        int     `db:"id" json:"id"`
-	Name      string  `db:"name" json:"name" binding:"required"`
-	ProjectID int     `db:"project_id" json:"project_id"`
-	Password  *string `db:"password" json:"password"`
-	JSON      string  `db:"json" json:"json" binding:"required"`
-	ENV       *string `db:"env" json:"env" binding:"required"`
+	ID        int                 `db:"id" json:"id"`
+	Name      string              `db:"name" json:"name" binding:"required"`
+	ProjectID int                 `db:"project_id" json:"project_id"`
+	Password  *string             `db:"password" json:"password"`
+	JSON      string              `db:"json" json:"json" binding:"required"`
+	ENV       *string             `db:"env" json:"env" binding:"required"`
+	Secrets   []EnvironmentSecret `db:"-" json:"secrets"`
 }
 
 func (env *Environment) Validate() error {

db/Migration.go

@@ -69,6 +69,7 @@ func GetMigrations() []Migration {
 		{Version: "2.9.97"},
 		{Version: "2.9.100"},
 		{Version: "2.10.12"},
+		{Version: "2.10.15"},
 	}
 }
 

db/Store.go

@@ -118,6 +118,7 @@ type Store interface {
 	UpdateEnvironment(env Environment) error
 	CreateEnvironment(env Environment) (Environment, error)
 	DeleteEnvironment(projectID int, templateID int) error
+	GetEnvironmentSecrets(projectID int, environmentID int) ([]AccessKey, error)
 
 	GetInventory(projectID int, inventoryID int) (Inventory, error)
 	GetInventoryRefs(projectID int, inventoryID int) (ObjectReferrers, error)

db/bolt/access_key.go

@@ -20,7 +20,10 @@ func (d *BoltDb) GetAccessKeyRefs(projectID int, accessKeyID int) (db.ObjectRefe
 
 func (d *BoltDb) GetAccessKeys(projectID int, params db.RetrieveQueryParams) ([]db.AccessKey, error) {
 	var keys []db.AccessKey
-	err := d.getObjects(projectID, db.AccessKeyProps, params, nil, &keys)
+	err := d.getObjects(projectID, db.AccessKeyProps, params, func(i interface{}) bool {
+		k := i.(db.AccessKey)
+		return k.EnvironmentID == nil
+	}, &keys)
 	return keys, err
 }
 

db/bolt/environment.go

@@ -40,3 +40,12 @@ func (d *BoltDb) CreateEnvironment(env db.Environment) (db.Environment, error) {
 func (d *BoltDb) DeleteEnvironment(projectID int, environmentID int) error {
 	return d.deleteObject(projectID, db.EnvironmentProps, intObjectID(environmentID), nil)
 }
+
+func (d *BoltDb) GetEnvironmentSecrets(projectID int, environmentID int) ([]db.AccessKey, error) {
+	var keys []db.AccessKey
+	err := d.getObjects(projectID, db.AccessKeyProps, db.RetrieveQueryParams{}, func(i interface{}) bool {
+		k := i.(db.AccessKey)
+		return k.EnvironmentID != nil && *k.EnvironmentID == environmentID
+	}, &keys)
+	return keys, err
+}

db/sql/SqlDb.go

@@ -202,7 +202,7 @@ func (d *SqlDb) getObject(projectID int, props db.ObjectProps, objectID int, obj
 	return
 }
 
-func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
+func (d *SqlDb) makeObjectsQuery(projectID int, props db.ObjectProps, params db.RetrieveQueryParams) squirrel.SelectBuilder {
 	q := squirrel.Select("*").
 		From(props.TableName + " pe")
 
@@ -232,7 +232,11 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.Retrie
 		q = q.Offset(uint64(params.Offset))
 	}
 
-	query, args, err := q.ToSql()
+	return q
+}
+
+func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
+	query, args, err := d.makeObjectsQuery(projectID, props, params).ToSql()
 
 	if err != nil {
 		return
@@ -243,10 +247,6 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.Retrie
 	return
 }
 
-func (d *SqlDb) getProjectObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
-	return d.getObjects(projectID, props, params, objects)
-}
-
 func (d *SqlDb) deleteObject(projectID int, props db.ObjectProps, objectID int) error {
 	if props.IsGlobal {
 		return validateMutationResult(

db/sql/access_key.go

@@ -15,10 +15,20 @@ func (d *SqlDb) GetAccessKeyRefs(projectID int, keyID int) (db.ObjectReferrers,
 	return d.getObjectRefs(projectID, db.AccessKeyProps, keyID)
 }
 
-func (d *SqlDb) GetAccessKeys(projectID int, params db.RetrieveQueryParams) ([]db.AccessKey, error) {
-	var keys []db.AccessKey
-	err := d.getProjectObjects(projectID, db.AccessKeyProps, params, &keys)
-	return keys, err
+func (d *SqlDb) GetAccessKeys(projectID int, params db.RetrieveQueryParams) (keys []db.AccessKey, err error) {
+	keys = make([]db.AccessKey, 0)
+
+	q := d.makeObjectsQuery(projectID, db.AccessKeyProps, params).Where("pe.environment_id IS NULL")
+
+	query, args, err := q.ToSql()
+
+	if err != nil {
+		return
+	}
+
+	_, err = d.selectAll(&keys, query, args...)
+
+	return
 }
 
 func (d *SqlDb) UpdateAccessKey(key db.AccessKey) error {
@@ -65,11 +75,12 @@ func (d *SqlDb) CreateAccessKey(key db.AccessKey) (newKey db.AccessKey, err erro
 
 	insertID, err := d.insert(
 		"id",
-		"insert into access_key (name, type, project_id, secret) values (?, ?, ?, ?)",
+		"insert into access_key (name, type, project_id, secret, environment_id) values (?, ?, ?, ?, ?)",
 		key.Name,
 		key.Type,
 		key.ProjectID,
-		key.Secret)
+		key.Secret,
+		key.EnvironmentID)
 
 	if err != nil {
 		return

db/sql/environment.go

@@ -15,7 +15,7 @@ func (d *SqlDb) GetEnvironmentRefs(projectID int, environmentID int) (db.ObjectR
 
 func (d *SqlDb) GetEnvironments(projectID int, params db.RetrieveQueryParams) ([]db.Environment, error) {
 	var environment []db.Environment
-	err := d.getProjectObjects(projectID, db.EnvironmentProps, params, &environment)
+	err := d.getObjects(projectID, db.EnvironmentProps, params, &environment)
 	return environment, err
 }
 
@@ -64,3 +64,19 @@ func (d *SqlDb) CreateEnvironment(env db.Environment) (newEnv db.Environment, er
 func (d *SqlDb) DeleteEnvironment(projectID int, environmentID int) error {
 	return d.deleteObject(projectID, db.EnvironmentProps, environmentID)
 }
+
+func (d *SqlDb) GetEnvironmentSecrets(projectID int, environmentID int) (keys []db.AccessKey, err error) {
+	keys = make([]db.AccessKey, 0)
+
+	q := d.makeObjectsQuery(projectID, db.AccessKeyProps, db.RetrieveQueryParams{}).Where("pe.environment_id = ?", environmentID)
+
+	query, args, err := q.ToSql()
+
+	if err != nil {
+		return
+	}
+
+	_, err = d.selectAll(&keys, query, args...)
+
+	return
+}

db/sql/integration.go

@@ -37,7 +37,7 @@ func (d *SqlDb) CreateIntegration(integration db.Integration) (newIntegration db
 }
 
 func (d *SqlDb) GetIntegrations(projectID int, params db.RetrieveQueryParams) (integrations []db.Integration, err error) {
-	err = d.getProjectObjects(projectID, db.IntegrationProps, params, &integrations)
+	err = d.getObjects(projectID, db.IntegrationProps, params, &integrations)
 	return integrations, err
 }
 

db/sql/inventory.go

@@ -14,7 +14,7 @@ func (d *SqlDb) GetInventory(projectID int, inventoryID int) (inventory db.Inven
 
 func (d *SqlDb) GetInventories(projectID int, params db.RetrieveQueryParams) ([]db.Inventory, error) {
 	var inventories []db.Inventory
-	err := d.getProjectObjects(projectID, db.InventoryProps, params, &inventories)
+	err := d.getObjects(projectID, db.InventoryProps, params, &inventories)
 	return inventories, err
 }
 

db/sql/migrations/v2.10.15.sql

@@ -0,0 +1 @@
+alter table `access_key` add `environment_id` int null references project__environment(`id`) on delete set null;

db/sql/runner.go

@@ -24,7 +24,7 @@ func (d *SqlDb) GetGlobalRunner(runnerID int) (runner db.Runner, err error) {
 }
 
 func (d *SqlDb) GetGlobalRunners() (runners []db.Runner, err error) {
-	err = d.getProjectObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, &runners)
+	err = d.getObjects(0, db.GlobalRunnerProps, db.RetrieveQueryParams{}, &runners)
 	return
 }
 

db/sql/view.go

@@ -8,7 +8,7 @@ func (d *SqlDb) GetView(projectID int, viewID int) (view db.View, err error) {
 }
 
 func (d *SqlDb) GetViews(projectID int) (views []db.View, err error) {
-	err = d.getProjectObjects(projectID, db.ViewProps, db.RetrieveQueryParams{}, &views)
+	err = d.getObjects(projectID, db.ViewProps, db.RetrieveQueryParams{}, &views)
 	return
 }
 

services/tasks/LocalJob.go

@@ -183,6 +183,10 @@ func (t *LocalJob) getBashArgs(username string, incomingVersion *string) (args [
 		args = append(args, fmt.Sprintf("%s=%s", name, value))
 	}
 
+	for _, secret := range t.Environment.Secrets {
+		args = append(args, fmt.Sprintf("%s=%s", secret.Name, secret.Secret))
+	}
+
 	return
 }
 
@@ -206,6 +210,10 @@ func (t *LocalJob) getTerraformArgs(username string, incomingVersion *string) (a
 		args = append(args, "-var", fmt.Sprintf("%s=%s", name, value))
 	}
 
+	for _, secret := range t.Environment.Secrets {
+		args = append(args, "-var", fmt.Sprintf("%s=%s", secret.Name, secret.Secret))
+	}
+
 	return
 }
 
@@ -302,6 +310,10 @@ func (t *LocalJob) getPlaybookArgs(username string, incomingVersion *string) (ar
 		args = append(args, "--extra-vars", extraVars)
 	}
 
+	for _, secret := range t.Environment.Secrets {
+		args = append(args, "--extra-vars", fmt.Sprintf("%s=%s", secret.Name, secret.Secret))
+	}
+
 	var templateExtraArgs []string
 	if t.Template.Arguments != nil {
 		err = json.Unmarshal([]byte(*t.Template.Arguments), &templateExtraArgs)

services/tasks/TaskRunner.go

@@ -274,6 +274,24 @@ func (t *TaskRunner) populateDetails() error {
 		if err != nil {
 			return err
 		}
+		var secrets []db.AccessKey
+		secrets, err = t.pool.store.GetEnvironmentSecrets(t.Template.ProjectID, *t.Template.EnvironmentID)
+		if err != nil {
+			return err
+		}
+
+		for _, s := range secrets {
+			err = s.DeserializeSecret()
+			if err != nil {
+				return err
+			}
+			t.Environment.Secrets = append(t.Environment.Secrets, db.EnvironmentSecret{
+				ID:     s.ID,
+				Name:   s.Name,
+				Secret: s.String,
+			})
+		}
+
 	}
 
 	if t.Task.Environment != "" {

web/src/components/EnvironmentForm.vue

@@ -23,7 +23,7 @@
     <v-subheader class="px-0">
       {{ $t('extraVariables') }}
 
-      <v-tooltip bottom color="black" open-delay="300">
+      <v-tooltip bottom color="black" open-delay="300" max-width="400">
         <template v-slot:activator="{ on, attrs }">
           <v-icon
             class="ml-1"
@@ -31,7 +31,10 @@
             v-on="on"
           >mdi-help-circle</v-icon>
         </template>
-        <span>Variables passed via <code>--extra-vars</code>.</span>
+        <span>
+          Variables passed via <code>--extra-vars</code> (Ansible) or
+          <code>-var</code> (Terraform/OpenTofu).
+        </span>
       </v-tooltip>
 
       <v-spacer />
@@ -55,7 +58,6 @@
     />
 
     <div>
-
       <v-subheader class="px-0 mt-4">
         {{ $t('environmentVariables') }}
 
@@ -71,7 +73,6 @@
           <span>Variables passed as process environment variables.</span>
         </v-tooltip>
       </v-subheader>
-
       <v-data-table
         :items="env"
         :items-per-page="-1"
@@ -111,12 +112,81 @@
           </tr>
         </template>
       </v-data-table>
-
-      <div class="text-right mt-2 mb-4">
+      <div class="mt-2 mb-4 mx-1">
         <v-btn
           color="primary"
           @click="addEnvVar()"
-        >New Variable</v-btn>
+        >New Environment Variable</v-btn>
+      </div>
+    </div>
+
+    <div>
+      <v-subheader class="px-0 mt-4">
+        {{ $t('Secrets') }}
+
+        <v-tooltip bottom color="black" open-delay="300" max-width="400">
+          <template v-slot:activator="{ on, attrs }">
+            <v-icon
+              class="ml-1"
+              v-bind="attrs"
+              v-on="on"
+              color="lightgray"
+            >mdi-help-circle</v-icon>
+          </template>
+          <span>
+            Secrets are stored in the database in encrypted form.
+            Secrets passed via <code>--extra-vars</code> (Ansible) or
+            <code>-var</code> (Terraform/OpenTofu).
+          </span>
+        </v-tooltip>
+      </v-subheader>
+
+      <v-data-table
+        :items="secrets.filter(s => !s.remove)"
+        :items-per-page="-1"
+        class="elevation-1"
+        hide-default-footer
+        no-data-text="No values"
+      >
+        <template v-slot:item="props">
+          <tr>
+            <td class="pa-1">
+              <v-text-field
+                solo-inverted
+                flat
+                hide-details
+                v-model="props.item.name"
+                class="v-text-field--solo--no-min-height"
+              ></v-text-field>
+            </td>
+            <td class="pa-1">
+              <v-text-field
+                solo-inverted
+                flat
+                hide-details
+                v-model="props.item.value"
+                placeholder="*******"
+                class="v-text-field--solo--no-min-height"
+              ></v-text-field>
+            </td>
+            <td style="width: 38px;">
+              <v-icon
+                small
+                class="pa-1"
+                @click="removeSecret(props.item)"
+              >
+                mdi-delete
+              </v-icon>
+            </td>
+          </tr>
+        </template>
+      </v-data-table>
+
+      <div class="mt-2 mb-4 mx-1">
+        <v-btn
+          color="primary"
+          @click="addSecret()"
+        >New Secret</v-btn>
       </div>
     </div>
 
@@ -157,8 +227,10 @@ export default {
         'dind-runner:latest',
       ],
       advancedOptions: false,
+
       json: '{}',
       env: [],
+      secrets: [],
 
       cmOptions: {
         tabSize: 2,
@@ -186,6 +258,25 @@ export default {
       }
     },
 
+    addSecret(name = '', value = '') {
+      this.secrets.push({ name, value, new: true });
+    },
+
+    removeSecret(val) {
+      const i = this.secrets.findIndex((v) => v.name === val.name);
+      if (i > -1) {
+        const s = this.secrets[i];
+        this.secrets.splice(i, 1);
+
+        if (!this.secrets[i].new) {
+          this.secrets.push({
+            ...s,
+            remove: true,
+          });
+        }
+      }
+    },
+
     setExtraVar(name, value) {
       try {
         const obj = JSON.parse(this.json || '{}');
@@ -212,7 +303,25 @@ export default {
         env[predefinedVar.name] = predefinedVar.value;
       });
 
+      const secrets = (this.secrets || []).map((s) => {
+        let operation;
+        if (s.new) {
+          operation = 'create';
+        } else if (s.remove) {
+          operation = 'delete';
+        } else if (s.value !== '') {
+          operation = 'update';
+        }
+        return {
+          id: s.id,
+          name: s.name,
+          secret: s.value,
+          operation,
+        };
+      }).filter((s) => s.operation != null);
+
       this.item.env = JSON.stringify(env);
+      this.item.secrets = secrets;
     },
 
     afterLoadData() {
@@ -220,6 +329,8 @@ export default {
 
       const env = JSON.parse(this.item?.env || '{}');
 
+      const secrets = this.item?.secrets || [];
+
       this.env = Object.keys(env)
         .filter((x) => {
           const index = PREDEFINED_ENV_VARS.findIndex((v) => v.name === x);
@@ -230,6 +341,12 @@ export default {
           value: env[x],
         }));
 
+      this.secrets = secrets.map((x) => ({
+        id: x.id,
+        name: x.name,
+        value: '',
+      }));
+
       Object.keys(env).forEach((x) => {
         const index = PREDEFINED_ENV_VARS.findIndex((v) => v.name === x);
         if (index !== -1 && PREDEFINED_ENV_VARS[index].value === env[x]) {