Merge remote-tracking branch 'real/develop' into develop

api-docs.yml

@@ -799,12 +799,21 @@ definitions:
   TemplateVault:
     type: object
     properties:
-      vault_key_id:
-        type: integer
-        minimum: 1
       name:
         type: string
         example: default
+      type:
+        type: string
+        enum: [password, script]
+        example: script
+      vault_key_id:
+        type:
+          - integer
+          - 'null'
+      script:
+        type:
+          - string
+          - 'null'
 
   ScheduleRequest:
     type: object

api/projects/projects.go

@@ -161,8 +161,9 @@ func createDemoProject(projectID int, noneKeyID int, emptyEnvID int, store db.St
 	_, err = store.CreateTemplateVault(db.TemplateVault{
 		ProjectID:  projectID,
 		TemplateID: template.ID,
-		VaultKeyID: vaultKey.ID,
+		VaultKeyID: &vaultKey.ID,
 		Name:       nil,
+		Type:       "password",
 	})
 
 	if err != nil {
@@ -188,8 +189,9 @@ func createDemoProject(projectID int, noneKeyID int, emptyEnvID int, store db.St
 	_, err = store.CreateTemplateVault(db.TemplateVault{
 		ProjectID:  projectID,
 		TemplateID: template.ID,
-		VaultKeyID: vaultKey.ID,
+		VaultKeyID: &vaultKey.ID,
 		Name:       nil,
+		Type:       "password",
 	})
 
 	return

api/runners/runners.go

@@ -91,11 +91,13 @@ func GetRunner(w http.ResponseWriter, r *http.Request) {
 
 			if tsk.Template.Vaults != nil {
 				for _, vault := range tsk.Template.Vaults {
-					err := vault.Vault.DeserializeSecret()
-					if err != nil {
-						// TODO: return error
+					if vault.VaultKeyID != nil {
+						err := vault.Vault.DeserializeSecret()
+						if err != nil {
+							// TODO: return error
+						}
+						data.AccessKeys[*vault.VaultKeyID] = *vault.Vault
 					}
-					data.AccessKeys[vault.VaultKeyID] = *vault.Vault
 				}
 			}
 

db/AccessKey.go

@@ -75,6 +75,7 @@ type AccessKeyInstallation struct {
 	SSHAgent *ssh.Agent
 	Login    string
 	Password string
+	Script   string
 }
 
 func (key AccessKeyInstallation) Destroy() error {
@@ -121,13 +122,15 @@ func (key *AccessKey) Install(usage AccessKeyRole, logger task_logger.Logger) (i
 			installation.Login = key.SshKey.Login
 		}
 	case AccessKeyRoleAnsiblePasswordVault:
-		if key.Type != AccessKeyLoginPassword {
-			err = fmt.Errorf("access key type not supported for ansible user")
+		switch key.Type {
+		case AccessKeyLoginPassword:
+			installation.Password = key.LoginPassword.Password
+		default:
+			err = fmt.Errorf("access key type not supported for ansible password vault")
 		}
-		installation.Password = key.LoginPassword.Password
 	case AccessKeyRoleAnsibleBecomeUser:
 		if key.Type != AccessKeyLoginPassword {
-			err = fmt.Errorf("access key type not supported for ansible user")
+			err = fmt.Errorf("access key type not supported for ansible become user")
 		}
 		installation.Login = key.LoginPassword.Login
 		installation.Password = key.LoginPassword.Password

db/Migration.go

@@ -75,6 +75,7 @@ func GetMigrations() []Migration {
 		{Version: "2.10.26"},
 		{Version: "2.10.27"},
 		{Version: "2.10.28"},
+		{Version: "2.10.33"},
 	}
 }
 

db/TemplateVault.go

@@ -1,21 +1,32 @@
 package db
 
+type TemplateVaultType string
+
+const (
+	TemplateVaultPassword TemplateVaultType = "password"
+	TemplateVaultScript   TemplateVaultType = "script"
+)
+
 type TemplateVault struct {
-	ID         int     `db:"id" json:"id" backup:"-"`
-	ProjectID  int     `db:"project_id" json:"project_id" backup:"-"`
-	TemplateID int     `db:"template_id" json:"template_id" backup:"-"`
-	VaultKeyID int     `db:"vault_key_id" json:"vault_key_id" backup:"-"`
-	Name       *string `db:"name" json:"name"`
+	ID         int               `db:"id" json:"id" backup:"-"`
+	ProjectID  int               `db:"project_id" json:"project_id" backup:"-"`
+	TemplateID int               `db:"template_id" json:"template_id" backup:"-"`
+	VaultKeyID *int              `db:"vault_key_id" json:"vault_key_id" backup:"-"`
+	Name       *string           `db:"name" json:"name"`
+	Type       TemplateVaultType `db:"type" json:"type"`
+	Script     *string           `db:"script" json:"script"`
 
 	Vault *AccessKey `db:"-" json:"-"`
 }
 
 func FillTemplateVault(d Store, projectID int, templateVault *TemplateVault) (err error) {
-	var vault AccessKey
-	vault, err = d.GetAccessKey(projectID, templateVault.VaultKeyID)
-	if err != nil {
-		return
+	if templateVault.Type == TemplateVaultPassword && templateVault.VaultKeyID != nil {
+		var vault AccessKey
+		vault, err = d.GetAccessKey(projectID, *templateVault.VaultKeyID)
+		if err != nil {
+			return
+		}
+		templateVault.Vault = &vault
 	}
-	templateVault.Vault = &vault
 	return
 }

db/bolt/migration.go

@@ -47,6 +47,8 @@ func (d *BoltDb) ApplyMigration(m db.Migration) (err error) {
 		err = migration_2_10_16{migration{d.db}}.Apply()
 	case "2.10.24":
 		err = migration_2_10_24{migration{d.db}}.Apply()
+	case "2.10.33":
+		err = migration_2_10_33{migration{d.db}}.Apply()
 	}
 
 	if err != nil {

db/bolt/migration_2_10_33.go

@@ -0,0 +1,38 @@
+package bolt
+
+type migration_2_10_33 struct {
+	migration
+}
+
+func (d migration_2_10_33) Apply() (err error) {
+	projectIDs, err := d.getProjectIDs()
+
+	if err != nil {
+		return
+	}
+
+	vaults := make(map[string]map[string]map[string]interface{})
+
+	for _, projectID := range projectIDs {
+		var err2 error
+		vaults[projectID], err2 = d.getObjects(projectID, "template_vault")
+		if err2 != nil {
+			return err2
+		}
+	}
+
+	for projectID, projectVaults := range vaults {
+		for repoID, vault := range projectVaults {
+			if vault["type"] != nil && vault["type"] != "" {
+				continue
+			}
+			vault["type"] = "password"
+			err = d.setObject(projectID, "template_vault", repoID, vault)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return
+}

db/bolt/migration_2_10_33_test.go

@@ -0,0 +1,84 @@
+package bolt
+
+import (
+	"encoding/json"
+	"go.etcd.io/bbolt"
+	"testing"
+)
+
+func TestMigration_2_10_33_Apply(t *testing.T) {
+	store := CreateTestStore()
+
+	err := store.db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("project"))
+		if err != nil {
+			return err
+		}
+
+		err = b.Put([]byte("0000000001"), []byte("{}"))
+		if err != nil {
+			return err
+		}
+
+		r, err := tx.CreateBucketIfNotExists([]byte("project__template_vault_0000000001"))
+		if err != nil {
+			return err
+		}
+
+		err = r.Put([]byte("0000000001"),
+			[]byte("{\"id\":\"1\",\"project_id\":\"1\"}"))
+
+		return err
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = migration_2_10_33{migration{store.db}}.Apply()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var repo map[string]interface{}
+	err = store.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte("project__template_vault_0000000001"))
+		str := string(b.Get([]byte("0000000001")))
+		return json.Unmarshal([]byte(str), &repo)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if repo["type"] == nil {
+		t.Fatal("app must be set")
+	}
+
+	if repo["type"].(string) != "password" {
+		t.Fatal("invalid app: " + repo["type"].(string))
+	}
+}
+
+func TestMigration_2_10_33_Apply2(t *testing.T) {
+	store := CreateTestStore()
+
+	err := store.db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("project"))
+		if err != nil {
+			return err
+		}
+
+		err = b.Put([]byte("0000000001"), []byte("{}"))
+
+		return err
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = migration_2_10_33{migration{store.db}}.Apply()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

db/bolt/template_vault.go

@@ -50,6 +50,14 @@ func (d *BoltDb) UpdateTemplateVaults(projectID int, templateID int, vaults []db
 		for _, vault := range vaults {
 			vault.ProjectID = projectID
 			vault.TemplateID = templateID
+
+			switch vault.Type {
+			case "password":
+				vault.Script = nil
+			case "script":
+				vault.VaultKeyID = nil
+			}
+
 			_, err = d.createObjectTx(tx, projectID, db.TemplateVaultProps, vault)
 			if err != nil {
 				return err

db/sql/migrations/v2.10.33.sql

@@ -0,0 +1,4 @@
+alter table `project__template_vault` change `vault_key_id` `vault_key_id` int;
+alter table `project__template_vault` add `type` varchar(20) not null default 'password';
+alter table `project__template_vault` add `script` text;
+update `project__template_vault` set `type` = 'password' where `vault_key_id` IS NOT NULL;

db/sql/template.go

@@ -241,6 +241,11 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
 			return
 		}
 
+		template.Vaults, err = d.GetTemplateVaults(projectID, template.ID)
+		if err != nil {
+			return
+		}
+
 		templates = append(templates, template)
 	}
 

db/sql/template_vault.go

@@ -25,11 +25,13 @@ func (d *SqlDb) GetTemplateVaults(projectID int, templateID int) (vaults []db.Te
 func (d *SqlDb) CreateTemplateVault(vault db.TemplateVault) (newVault db.TemplateVault, err error) {
 	insertID, err := d.insert(
 		"id",
-		"insert into project__template_vault (project_id, template_id, vault_key_id, name) values (?, ?, ?, ?)",
+		"insert into project__template_vault (project_id, template_id, vault_key_id, name, type, script) values (?, ?, ?, ?, ?, ?)",
 		vault.ProjectID,
 		vault.TemplateID,
 		vault.VaultKeyID,
-		vault.Name)
+		vault.Name,
+		vault.Type,
+		vault.Script)
 	if err != nil {
 		return
 	}
@@ -46,17 +48,23 @@ func (d *SqlDb) UpdateTemplateVaults(projectID int, templateID int, vaults []db.
 
 	var vaultIDs []string
 	for _, vault := range vaults {
+		switch vault.Type {
+		case "password":
+			vault.Script = nil
+		case "script":
+			vault.VaultKeyID = nil
+		}
 		if vault.ID == 0 {
 			// Insert new vaults
 			var vaultId int
-			vaultId, err = d.insert("id", "insert into project__template_vault (project_id, template_id, vault_key_id, name) values (?, ?, ?, ?)", projectID, templateID, vault.VaultKeyID, vault.Name)
+			vaultId, err = d.insert("id", "insert into project__template_vault (project_id, template_id, vault_key_id, name, type, script) values (?, ?, ?, ?, ?, ?)", projectID, templateID, vault.VaultKeyID, vault.Name, vault.Type, vault.Script)
 			if err != nil {
 				return
 			}
 			vaultIDs = append(vaultIDs, strconv.Itoa(vaultId))
 		} else {
 			// Update existing vaults
-			_, err = d.exec("update project__template_vault set project_id=?, template_id=?, vault_key_id=?, name=? where id=?", projectID, templateID, vault.VaultKeyID, vault.Name, vault.ID)
+			_, err = d.exec("update project__template_vault set project_id=?, template_id=?, vault_key_id=?, name=?, type=?, script=? where id=?", projectID, templateID, vault.VaultKeyID, vault.Name, vault.Type, vault.Script, vault.ID)
 			vaultIDs = append(vaultIDs, strconv.Itoa(vault.ID))
 		}
 		if err != nil {

db_lib/AnsiblePlaybook.go

@@ -24,7 +24,6 @@ func (p AnsiblePlaybook) makeCmd(command string, args []string, environmentVars
 
 	cmd.Env = append(cmd.Env, "PYTHONUNBUFFERED=1")
 	cmd.Env = append(cmd.Env, "ANSIBLE_FORCE_COLOR=True")
-	cmd.Env = append(cmd.Env, fmt.Sprintf("PATH=%s", os.Getenv("PATH")))
 	cmd.Env = append(cmd.Env, getEnvironmentVars()...)
 	cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", util.Config.TmpPath))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("PWD=%s", cmd.Dir))

db_lib/LocalApp.go

@@ -9,7 +9,9 @@ import (
 )
 
 func getEnvironmentVars() []string {
-	res := []string{}
+	res := []string{
+		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
+	}
 
 	for _, e := range util.Config.ForwardedEnvVars {
 		v := os.Getenv(e)

db_lib/LocalApp_test.go

@@ -2,6 +2,7 @@ package db_lib
 
 import (
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/semaphoreui/semaphore/util"
@@ -10,7 +11,7 @@ import (
 // contains checks if a slice contains a specific string
 func contains(slice []string, item string) bool {
 	for _, s := range slice {
-		if s == item {
+		if strings.HasPrefix(s, item) {
 			return true
 		}
 	}
@@ -35,6 +36,7 @@ func TestGetEnvironmentVars(t *testing.T) {
 	expected := []string{
 		"SEMAPHORE_TEST=test123",
 		"ANSIBLE_FORCE_COLOR=False",
+		"PATH=",
 	}
 
 	if len(res) != len(expected) {