From 421e862786531ef027d8bad9b768fd6cd2889e09 Mon Sep 17 00:00:00 2001 From: AnsibleGuy Date: Sat, 5 Aug 2023 15:56:39 +0200 Subject: [PATCH 01/10] feat: added basic config validation, loading all settings from environment-variables, dynamic applying of default-values to settings, tests for config-loading and -validation --- db/AccessKey.go | 4 +- util/config.go | 237 +++++++++++++++++++++++++++++++------ util/config_test.go | 280 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 469 insertions(+), 52 deletions(-) diff --git a/db/AccessKey.go b/db/AccessKey.go index a0c93f03..65e14b84 100644 --- a/db/AccessKey.go +++ b/db/AccessKey.go @@ -198,7 +198,7 @@ func (key *AccessKey) SerializeSecret() error { return fmt.Errorf("invalid access token type") } - encryptionString := util.Config.GetAccessKeyEncryption() + encryptionString := util.Config.AccessKeyEncryption if encryptionString == "" { secret := base64.StdEncoding.EncodeToString(plaintext) @@ -279,7 +279,7 @@ func (key *AccessKey) DeserializeSecret() error { return err } - encryptionString := util.Config.GetAccessKeyEncryption() + encryptionString := util.Config.AccessKeyEncryption if encryptionString == "" { err = key.unmarshalAppropriateField(ciphertext) diff --git a/util/config.go b/util/config.go index 3946e2f4..0061c6dd 100644 --- a/util/config.go +++ b/util/config.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/google/go-github/github" "io" "net/url" "os" @@ -14,7 +13,11 @@ import ( "path" "path/filepath" "strings" + "reflect" + "regexp" + "strconv" + "github.com/google/go-github/github" "github.com/gorilla/securecookie" ) @@ -115,7 +118,6 @@ type ConfigType struct { CookieEncryption string `json:"cookie_encryption"` // AccessKeyEncryption is BASE64 encoded byte array used // for encrypting and decrypting access keys stored in database. - // Do not use it! Use method GetAccessKeyEncryption instead of it. AccessKeyEncryption string `json:"access_key_encryption"` // email alerting @@ -159,25 +161,96 @@ type ConfigType struct { // Config exposes the application configuration storage for use in the application var Config *ConfigType +var ( + // default config values + configDefaults = map[string]interface{}{ + "Port": ":3000", + "TmpPath": "/tmp/semaphore", + "GitClientId": GoGitClientId, + } + + // mapping internal config to env-vars + // todo: special cases - SEMAPHORE_DB_PORT, SEMAPHORE_DB_PATH (bolt), SEMAPHORE_CONFIG_PATH, OPENID for 1 provider if it makes sense + ConfigEnvironmentalVars = map[string]string{ + "Dialect": "SEMAPHORE_DB_DIALECT", + "MySQL.Hostname": "SEMAPHORE_DB_HOST", + "MySQL.Username": "SEMAPHORE_DB_USER", + "MySQL.Password": "SEMAPHORE_DB_PASS", + "MySQL.DbName": "SEMAPHORE_DB", + "Postgres.Hostname": "SEMAPHORE_DB_HOST", + "Postgres.Username": "SEMAPHORE_DB_USER", + "Postgres.Password": "SEMAPHORE_DB_PASS", + "Postgres.DbName": "SEMAPHORE_DB", + "BoltDb.Hostname": "SEMAPHORE_DB_HOST", + "Port": "SEMAPHORE_PORT", + "Interface": "SEMAPHORE_INTERFACE", + "TmpPath": "SEMAPHORE_TMP_PATH", + "SshConfigPath": "SEMAPHORE_TMP_PATH", + "GitClientId": "SEMAPHORE_GIT_CLIENT", + "WebHost": "SEMAPHORE_WEB_ROOT", + "CookieHash": "SEMAPHORE_COOKIE_HASH", + "CookieEncryption": "SEMAPHORE_COOKIE_ENCRYPTION", + "AccessKeyEncryption": "SEMAPHORE_ACCESS_KEY_ENCRYPTION", + "EmailAlert": "SEMAPHORE_EMAIL_ALERT", + "EmailSender": "SEMAPHORE_EMAIL_SENDER", + "EmailHost": "SEMAPHORE_EMAIL_HOST", + "EmailPort": "SEMAPHORE_EMAIL_PORT", + "EmailUsername": "SEMAPHORE_EMAIL_USER", + "EmailPassword": "SEMAPHORE_EMAIL_PASSWORD", + "EmailSecure": "SEMAPHORE_EMAIL_SECURE", + "LdapEnable": "SEMAPHORE_LDAP_ACTIVATED", + "LdapBindDN": "SEMAPHORE_LDAP_DN_BIND", + "LdapBindPassword": "SEMAPHORE_LDAP_PASSWORD", + "LdapServer": "SEMAPHORE_LDAP_HOST", + "LdapSearchDN": "SEMAPHORE_LDAP_DN_SEARCH", + "LdapSearchFilter": "SEMAPHORE_LDAP_SEARCH_FILTER", + "LdapMappings.DN": "SEMAPHORE_LDAP_MAPPING_DN", + "LdapMappings.UID": "SEMAPHORE_LDAP_MAPPING_USERNAME", + "LdapMappings.CN": "SEMAPHORE_LDAP_MAPPING_FULLNAME", + "LdapMappings.Mail": "SEMAPHORE_LDAP_MAPPING_EMAIL", + "LdapNeedTLS": "SEMAPHORE_LDAP_NEEDTLS", + "TelegramAlert": "SEMAPHORE_TELEGRAM_ALERT", + "TelegramChat": "SEMAPHORE_TELEGRAM_CHAT", + "TelegramToken": "SEMAPHORE_TELEGRAM_TOKEN", + "SlackAlert": "SEMAPHORE_SLACK_ALERT", + "SlackUrl": "SEMAPHORE_SLACK_URL", + "MaxParallelTasks": "SEMAPHORE_MAX_PARALLEL_TASKS", + } + + // basic config validation using regex + /* NOTE: other basic regex could be used: + ipv4: ^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$ + ipv6: ^(?:[A-Fa-f0-9]{1,4}:|:){3,7}[A-Fa-f0-9]{1,4}$ + domain: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,}$ + path+filename: ^([\\/[a-zA-Z0-9_\\-${}:~]*]*\\/)?[a-zA-Z0-9\\.~_${}\\-:]*$ + email address: ^(|.*@[A-Za-z0-9-\\.]*)$ + */ + configValidationRegex = map[string]string{ + "Dialect": "^mysql|bolt|postgres$", + "Port": "^:([0-9]{1,5})$", // can have false-negatives + "GitClientId": "^go_git|cmd_git$", + "CookieHash": "^[-A-Za-z0-9+=\\/]{40,}$", // base64 + "CookieEncryption": "^[-A-Za-z0-9+=\\/]{40,}$", // base64 + "AccessKeyEncryption": "^[-A-Za-z0-9+=\\/]{40,}$", // base64 + "EmailPort": "^(|[0-9]{1,5})$", // can have false-negatives + "MaxParallelTasks": "^[0-9]{1,10}$", // 0-9999999999 + } +) + // ToJSON returns a JSON string of the config func (conf *ConfigType) ToJSON() ([]byte, error) { return json.MarshalIndent(&conf, " ", "\t") } -func (conf *ConfigType) GetAccessKeyEncryption() string { - ret := os.Getenv("SEMAPHORE_ACCESS_KEY_ENCRYPTION") - - if ret == "" { - ret = conf.AccessKeyEncryption - } - - return ret -} - // ConfigInit reads in cli flags, and switches actions appropriately on them func ConfigInit(configPath string) { - loadConfig(configPath) - validateConfig() + fmt.Println("Loading config") + loadConfigFile(configPath) + loadConfigEnvironment() + loadConfigDefaults() + + fmt.Println("Validating config") + validateConfig(exitOnConfigError) var encryption []byte @@ -193,7 +266,7 @@ func ConfigInit(configPath string) { } } -func loadConfig(configPath string) { +func loadConfigFile(configPath string) { if configPath == "" { configPath = os.Getenv("SEMAPHORE_CONFIG_PATH") } @@ -203,7 +276,7 @@ func loadConfig(configPath string) { if configPath == "" { cwd, err := os.Getwd() - exitOnConfigError(err) + exitOnConfigFileError(err) paths := []string{ path.Join(cwd, "config.json"), "/usr/local/etc/semaphore/config.json", @@ -221,46 +294,140 @@ func loadConfig(configPath string) { decodeConfig(file) break } - exitOnConfigError(err) + exitOnConfigFileError(err) } else { p := configPath file, err := os.Open(p) - exitOnConfigError(err) + exitOnConfigFileError(err) decodeConfig(file) } } -func validateConfig() { +func loadConfigDefaults() { - validatePort() - - if len(Config.TmpPath) == 0 { - Config.TmpPath = "/tmp/semaphore" + for attribute, defaultValue := range configDefaults { + if len(getConfigValue(attribute)) == 0 { + setConfigValue(attribute, defaultValue) + } } - if Config.MaxParallelTasks < 1 { - Config.MaxParallelTasks = 10 - } } -func validatePort() { +func castStringToInt(value string) int { - //TODO - why do we do this only with this variable? - if len(os.Getenv("PORT")) > 0 { - Config.Port = ":" + os.Getenv("PORT") + valueInt, err := strconv.Atoi(value) + if err != nil { + panic(err) } - if len(Config.Port) == 0 { - Config.Port = ":3000" + return valueInt + +} + +func castStringToBool(value string) bool { + + var valueBool bool + if value == "1" || strings.ToLower(value) == "true" { + valueBool = true + } else { + valueBool = false } + return valueBool + +} + +func setConfigValue(path string, value interface{}) { + + attribute := reflect.ValueOf(Config) + + for _, nested := range strings.Split(path, ".") { + attribute = reflect.Indirect(attribute).FieldByName(nested) + } + if attribute.IsValid() { + switch attribute.Kind() { + case reflect.Int: + if reflect.ValueOf(value).Kind() != reflect.Int { + value = castStringToInt(fmt.Sprintf("%v", reflect.ValueOf(value))) + } + case reflect.Bool: + if reflect.ValueOf(value).Kind() != reflect.Bool { + value = castStringToBool(fmt.Sprintf("%v", reflect.ValueOf(value))) + } + } + attribute.Set(reflect.ValueOf(value)) + } + +} + +func getConfigValue(path string) string { + + attribute := reflect.ValueOf(Config) + nested_path := strings.Split(path, ".") + + for i, nested := range nested_path { + attribute = reflect.Indirect(attribute).FieldByName(nested) + lastDepth := len(nested_path) == i+1 + if !lastDepth && attribute.Kind() != reflect.Struct || lastDepth && attribute.Kind() == reflect.Invalid { + panic(fmt.Errorf("got non-existent config attribute '%v'", path)) + } + } + + return fmt.Sprintf("%v", attribute) +} + +func validateConfig(errorFunc func(string)) { + if !strings.HasPrefix(Config.Port, ":") { Config.Port = ":" + Config.Port } + + for attribute, validateRegex := range configValidationRegex { + value := getConfigValue(attribute) + match, _ := regexp.MatchString(validateRegex, value) + if !match { + if !strings.Contains(attribute, "assword") && !strings.Contains(attribute, "ecret") { + errorFunc(fmt.Sprintf( + "value of setting '%v' is not valid: '%v' (Must match regex: '%v')", + attribute, value, validateRegex, + )) + } else { + errorFunc(fmt.Sprintf( + "value of setting '%v' is not valid! (Must match regex: '%v')", + attribute, validateRegex, + )) + } + } + } + } -func exitOnConfigError(err error) { +func loadConfigEnvironment() { + + for attribute, envVar := range ConfigEnvironmentalVars { + // skip unused db-dialects as they use the same env-vars + if strings.Contains(attribute, "MySQL") && Config.Dialect != DbDriverMySQL { + continue + } else if strings.Contains(attribute, "Postgres") && Config.Dialect != DbDriverPostgres { + continue + } else if strings.Contains(attribute, "BoldDb") && Config.Dialect != DbDriverBolt { + continue + } + + envValue, exists := os.LookupEnv(envVar) + if exists && len(envValue) > 0 { + setConfigValue(attribute, envValue) + } + } + +} + +func exitOnConfigError(msg string) { + fmt.Println(msg) + os.Exit(1) +} + +func exitOnConfigFileError(err error) { if err != nil { - fmt.Println("Cannot Find configuration! Use --config parameter to point to a JSON file generated by `semaphore setup`.") - os.Exit(1) + exitOnConfigError("Cannot Find configuration! Use --config parameter to point to a JSON file generated by `semaphore setup`.") } } diff --git a/util/config_test.go b/util/config_test.go index 73bf866b..7bafcbb9 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -1,28 +1,278 @@ package util import ( + "fmt" "os" "testing" ) -func TestValidatePort(t *testing.T) { +func mockError(msg string) { + panic(msg) +} + +func TestCastStringToInt(t *testing.T) { + + var errMsg string = "Cast string => int failed" + + if castStringToInt("5") != 5 { + t.Error(errMsg) + } + if castStringToInt("0") != 0 { + t.Error(errMsg) + } + if castStringToInt("-1") != -1 { + t.Error(errMsg) + } + if castStringToInt("999") != 999 { + t.Error(errMsg) + } + if castStringToInt("999") != 999 { + t.Error(errMsg) + } + defer func() { + if r := recover(); r == nil { + t.Errorf("Cast string => int did not panic on invalid input") + } + }() + castStringToInt("xxx") + +} + +func TestCastStringToBool(t *testing.T) { + + var errMsg string = "Cast string => bool failed" + + if castStringToBool("1") != true { + t.Error(errMsg) + } + if castStringToBool("0") != false { + t.Error(errMsg) + } + if castStringToBool("true") != true { + t.Error(errMsg) + } + if castStringToBool("false") != false { + t.Error(errMsg) + } + if castStringToBool("xxx") != false { + t.Error(errMsg) + } + if castStringToBool("") != false { + t.Error(errMsg) + } + +} + +func TestGetConfigValue(t *testing.T) { Config = new(ConfigType) - Config.Port = "" - validatePort() + + var testPort string = "1337" + var testCookieHash string = "0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=" + var testMaxParallelTasks int = 5 + var testLdapNeedTls bool = true + var testDbHost string = "192.168.0.1" + + Config.Port = testPort + Config.CookieHash = testCookieHash + Config.MaxParallelTasks = testMaxParallelTasks + Config.LdapNeedTLS = testLdapNeedTls + Config.BoltDb.Hostname = testDbHost + + if getConfigValue("Port") != testPort { + t.Error("Could not get value for config attribute 'Port'!") + } + if getConfigValue("CookieHash") != testCookieHash { + t.Error("Could not get value for config attribute 'CookieHash'!") + } + if getConfigValue("MaxParallelTasks") != fmt.Sprintf("%v", testMaxParallelTasks) { + t.Error("Could not get value for config attribute 'MaxParallelTasks'!") + } + if getConfigValue("LdapNeedTLS") != fmt.Sprintf("%v", testLdapNeedTls) { + t.Error("Could not get value for config attribute 'LdapNeedTLS'!") + } + if getConfigValue("BoltDb.Hostname") != fmt.Sprintf("%v", testDbHost) { + t.Error("Could not get value for config attribute 'BoltDb.Hostname'!") + } + + defer func() { + if r := recover(); r == nil { + t.Error("Did not fail on non-existent config attribute!") + } + }() + + getConfigValue("Not.Existent") + +} + +func TestSetConfigValue(t *testing.T) { + + Config = new(ConfigType) + + var testPort string = "1337" + var testCookieHash string = "0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=" + var testMaxParallelTasks int = 5 + var testLdapNeedTls bool = true + var testDbHost string = "192.168.0.1" + var testEmailSecure string = "1" + var expectEmailSecure bool = true + + setConfigValue("Port", testPort) + setConfigValue("CookieHash", testCookieHash) + setConfigValue("MaxParallelTasks", testMaxParallelTasks) + setConfigValue("LdapNeedTLS", testLdapNeedTls) + setConfigValue("BoltDb.Hostname", testDbHost) + setConfigValue("EmailSecure", testEmailSecure) + + if Config.Port != testPort { + t.Error("Could not set value for config attribute 'Port'!") + } + if Config.CookieHash != testCookieHash { + t.Error("Could not set value for config attribute 'CookieHash'!") + } + if Config.MaxParallelTasks != testMaxParallelTasks { + t.Error("Could not set value for config attribute 'MaxParallelTasks'!") + } + if Config.LdapNeedTLS != testLdapNeedTls { + t.Error("Could not set value for config attribute 'LdapNeedTls'!") + } + if Config.BoltDb.Hostname != testDbHost { + t.Error("Could not set value for config attribute 'BoltDb.Hostname'!") + } + if Config.EmailSecure != expectEmailSecure { + t.Error("Could not set value for config attribute 'EmailSecure'!") + } + + defer func() { + if r := recover(); r == nil { + t.Error("Did not fail on non-existent config attribute!") + } + }() + + setConfigValue("Not.Existent", "someValue") + +} + +func TestLoadConfigEnvironmet(t *testing.T) { + + Config = new(ConfigType) + Config.Dialect = DbDriverBolt + + var envPort string = "1337" + var envCookieHash string = "0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=" + var envAccessKeyEncryption string = "1/wRYXQltDGwbzNZRP9ZfJb2IoWcn1hYrxA0vOdvVos=" + var envMaxParallelTasks string = "5" + var expectMaxParallelTasks int = 5 + var expectLdapNeedTls bool = true + var envLdapNeedTls string = "1" + var envDbHost string = "192.168.0.1" + + os.Setenv("SEMAPHORE_PORT", envPort) + os.Setenv("SEMAPHORE_COOKIE_HASH", envCookieHash) + os.Setenv("SEMAPHORE_ACCESS_KEY_ENCRYPTION", envAccessKeyEncryption) + os.Setenv("SEMAPHORE_MAX_PARALLEL_TASKS", envMaxParallelTasks) + os.Setenv("SEMAPHORE_LDAP_NEEDTLS", envLdapNeedTls) + os.Setenv("SEMAPHORE_DB_HOST", envDbHost) + + loadConfigEnvironment() + + if Config.Port != envPort { + t.Error("Setting 'Port' was not loaded from environment-vars!") + } + if Config.CookieHash != envCookieHash { + t.Error("Setting 'CookieHash' was not loaded from environment-vars!") + } + if Config.AccessKeyEncryption != envAccessKeyEncryption { + t.Error("Setting 'AccessKeyEncryption' was not loaded from environment-vars!") + } + if Config.MaxParallelTasks != expectMaxParallelTasks { + t.Error("Setting 'MaxParallelTasks' was not loaded from environment-vars!") + } + if Config.LdapNeedTLS != expectLdapNeedTls { + t.Error("Setting 'LdapNeedTLS' was not loaded from environment-vars!") + } + if Config.BoltDb.Hostname != envDbHost { + t.Error("Setting 'BoltDb.Hostname' was not loaded from environment-vars!") + } + if Config.MySQL.Hostname == envDbHost || Config.Postgres.Hostname == envDbHost { + t.Error("DB-Hostname was not loaded for inactive DB-dialects!") + } + +} + +func TestLoadConfigDefaults(t *testing.T) { + + Config = new(ConfigType) + var errMsg string = "Failed to load config-default" + + loadConfigDefaults() + if Config.Port != ":3000" { - t.Error("no port should get set to default") + t.Error(errMsg) } - - Config.Port = "4000" - validatePort() - if Config.Port != ":4000" { - t.Error("Port without : suffix should have it added") - } - - os.Setenv("PORT", "5000") - validatePort() - if Config.Port != ":5000" { - t.Error("Port value should be overwritten by env var, and it should be prefixed appropriately") + if Config.TmpPath != "/tmp/semaphore" { + t.Error(errMsg) } } + +func ensureConfigValidationFailure(t *testing.T, attribute string, value interface{}) { + + defer func() { + if r := recover(); r == nil { + t.Errorf( + "Config validation for attribute '%v' did not fail! (value '%v')", + attribute, value, + ) + } + }() + validateConfig(mockError) + +} + +func TestValidateConfig(t *testing.T) { + + Config = new(ConfigType) + + var testPort string = ":3000" + var testDbDialect DbDriver = DbDriverBolt + var testCookieHash string = "0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=" + var testMaxParallelTasks int = 0 + + Config.Port = testPort + Config.Dialect = testDbDialect + Config.CookieHash = testCookieHash + Config.MaxParallelTasks = testMaxParallelTasks + Config.GitClientId = GoGitClientId + Config.CookieEncryption = testCookieHash + Config.AccessKeyEncryption = testCookieHash + validateConfig(mockError) + + Config.Port = "INVALID" + ensureConfigValidationFailure(t, "Port", Config.Port) + + Config.Port = ":100000" + ensureConfigValidationFailure(t, "Port", Config.Port) + Config.Port = testPort + + Config.MaxParallelTasks = -1 + ensureConfigValidationFailure(t, "MaxParallelTasks", Config.MaxParallelTasks) + Config.MaxParallelTasks = testMaxParallelTasks + + Config.CookieHash = "\"0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=\"" + ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + + Config.CookieHash = "!)394340" + ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + + Config.CookieHash = "" + ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + + Config.CookieHash = "TQwjDZ5fIQtaIw==" + ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + Config.CookieHash = testCookieHash + + Config.Dialect = "someOtherDB" + ensureConfigValidationFailure(t, "Dialect", Config.Dialect) + Config.Dialect = testDbDialect + +} From 07ee77d6dbe313eab286b67668f8e58f8648613f Mon Sep 17 00:00:00 2001 From: AnsibleGuy Date: Sun, 6 Aug 2023 11:01:24 +0200 Subject: [PATCH 02/10] feat: config-validation - minor fixes --- util/config.go | 4 +++- util/config_test.go | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/util/config.go b/util/config.go index 0061c6dd..efe0164b 100644 --- a/util/config.go +++ b/util/config.go @@ -12,10 +12,10 @@ import ( "os/exec" "path" "path/filepath" - "strings" "reflect" "regexp" "strconv" + "strings" "github.com/google/go-github/github" "github.com/gorilla/securecookie" @@ -354,6 +354,8 @@ func setConfigValue(path string, value interface{}) { } } attribute.Set(reflect.ValueOf(value)) + } else { + panic(fmt.Errorf("got non-existent config attribute '%v'", path)) } } diff --git a/util/config_test.go b/util/config_test.go index 7bafcbb9..e0d15940 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -26,9 +26,7 @@ func TestCastStringToInt(t *testing.T) { if castStringToInt("999") != 999 { t.Error(errMsg) } - if castStringToInt("999") != 999 { - t.Error(errMsg) - } + defer func() { if r := recover(); r == nil { t.Errorf("Cast string => int did not panic on invalid input") @@ -100,7 +98,13 @@ func TestGetConfigValue(t *testing.T) { t.Error("Did not fail on non-existent config attribute!") } }() + getConfigValue("NotExistent") + defer func() { + if r := recover(); r == nil { + t.Error("Did not fail on non-existent config attribute!") + } + }() getConfigValue("Not.Existent") } @@ -148,7 +152,13 @@ func TestSetConfigValue(t *testing.T) { t.Error("Did not fail on non-existent config attribute!") } }() + setConfigValue("NotExistent", "someValue") + defer func() { + if r := recover(); r == nil { + t.Error("Did not fail on non-existent config attribute!") + } + }() setConfigValue("Not.Existent", "someValue") } @@ -195,7 +205,8 @@ func TestLoadConfigEnvironmet(t *testing.T) { t.Error("Setting 'BoltDb.Hostname' was not loaded from environment-vars!") } if Config.MySQL.Hostname == envDbHost || Config.Postgres.Hostname == envDbHost { - t.Error("DB-Hostname was not loaded for inactive DB-dialects!") + // inactive db-dialects could be set as they share the same env-vars; but should be ignored + t.Error("DB-Hostname was loaded for inactive DB-dialects!") } } @@ -258,7 +269,7 @@ func TestValidateConfig(t *testing.T) { ensureConfigValidationFailure(t, "MaxParallelTasks", Config.MaxParallelTasks) Config.MaxParallelTasks = testMaxParallelTasks - Config.CookieHash = "\"0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=\"" + Config.CookieHash = "\"0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=\"" // invalid with quotes (can happen when supplied as env-var) ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) Config.CookieHash = "!)394340" @@ -267,7 +278,7 @@ func TestValidateConfig(t *testing.T) { Config.CookieHash = "" ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) - Config.CookieHash = "TQwjDZ5fIQtaIw==" + Config.CookieHash = "TQwjDZ5fIQtaIw==" // valid b64, but too small ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) Config.CookieHash = testCookieHash From cffba6e489201fb52e8af0c3a19a957b50cf01d9 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Sat, 9 Sep 2023 17:01:36 +0200 Subject: [PATCH 03/10] refactor(config): add tags to config fields --- util/config.go | 303 +++++++++++++++++++++++++------------------- util/config_test.go | 64 ++++++++-- 2 files changed, 229 insertions(+), 138 deletions(-) diff --git a/util/config.go b/util/config.go index efe0164b..865da577 100644 --- a/util/config.go +++ b/util/config.go @@ -38,10 +38,10 @@ const ( type DbConfig struct { Dialect DbDriver `json:"-"` - Hostname string `json:"host"` - Username string `json:"user"` - Password string `json:"pass"` - DbName string `json:"name"` + Hostname string `json:"host" env:"SEMAPHORE_DB_HOST"` + Username string `json:"user" env:"SEMAPHORE_DB_USER"` + Password string `json:"pass" env:"SEMAPHORE_DB_PASS"` + DbName string `json:"name" env:"SEMAPHORE_DB"` Options map[string]string `json:"options"` } @@ -85,46 +85,106 @@ const ( CmdGitClientId GitClientId = "cmd_git" ) +// // mapping internal config to env-vars +// // todo: special cases - SEMAPHORE_DB_PORT, SEMAPHORE_DB_PATH (bolt), SEMAPHORE_CONFIG_PATH, OPENID for 1 provider if it makes sense +// +// ConfigEnvironmentalVars = map[string]string{ +// "Dialect": "SEMAPHORE_DB_DIALECT", +// "MySQL.Hostname": "SEMAPHORE_DB_HOST", +// "MySQL.Username": "SEMAPHORE_DB_USER", +// "MySQL.Password": "SEMAPHORE_DB_PASS", +// "MySQL.DbName": "SEMAPHORE_DB", +// "Postgres.Hostname": "SEMAPHORE_DB_HOST", +// "Postgres.Username": "SEMAPHORE_DB_USER", +// "Postgres.Password": "SEMAPHORE_DB_PASS", +// "Postgres.DbName": "SEMAPHORE_DB", +// "BoltDb.Hostname": "SEMAPHORE_DB_HOST", +// "Port": "SEMAPHORE_PORT", +// "Interface": "SEMAPHORE_INTERFACE", +// "TmpPath": "SEMAPHORE_TMP_PATH", +// "SshConfigPath": "SEMAPHORE_TMP_PATH", +// "GitClientId": "SEMAPHORE_GIT_CLIENT", +// "WebHost": "SEMAPHORE_WEB_ROOT", +// "CookieHash": "SEMAPHORE_COOKIE_HASH", +// "CookieEncryption": "SEMAPHORE_COOKIE_ENCRYPTION", +// "AccessKeyEncryption": "SEMAPHORE_ACCESS_KEY_ENCRYPTION", +// "EmailAlert": "SEMAPHORE_EMAIL_ALERT", +// "EmailSender": "SEMAPHORE_EMAIL_SENDER", +// "EmailHost": "SEMAPHORE_EMAIL_HOST", +// "EmailPort": "SEMAPHORE_EMAIL_PORT", +// "EmailUsername": "SEMAPHORE_EMAIL_USER", +// "EmailPassword": "SEMAPHORE_EMAIL_PASSWORD", +// "EmailSecure": "SEMAPHORE_EMAIL_SECURE", +// "LdapEnable": "SEMAPHORE_LDAP_ACTIVATED", +// "LdapBindDN": "SEMAPHORE_LDAP_DN_BIND", +// "LdapBindPassword": "SEMAPHORE_LDAP_PASSWORD", +// "LdapServer": "SEMAPHORE_LDAP_HOST", +// "LdapSearchDN": "SEMAPHORE_LDAP_DN_SEARCH", +// "LdapSearchFilter": "SEMAPHORE_LDAP_SEARCH_FILTER", +// "LdapMappings.DN": "SEMAPHORE_LDAP_MAPPING_DN", +// "LdapMappings.UID": "SEMAPHORE_LDAP_MAPPING_USERNAME", +// "LdapMappings.CN": "SEMAPHORE_LDAP_MAPPING_FULLNAME", +// "LdapMappings.Mail": "SEMAPHORE_LDAP_MAPPING_EMAIL", +// "LdapNeedTLS": "SEMAPHORE_LDAP_NEEDTLS", +// "TelegramAlert": "SEMAPHORE_TELEGRAM_ALERT", +// "TelegramChat": "SEMAPHORE_TELEGRAM_CHAT", +// "TelegramToken": "SEMAPHORE_TELEGRAM_TOKEN", +// "SlackAlert": "SEMAPHORE_SLACK_ALERT", +// "SlackUrl": "SEMAPHORE_SLACK_URL", +// "MaxParallelTasks": "SEMAPHORE_MAX_PARALLEL_TASKS", +// } +// +// // basic config validation using regex +// /* NOTE: other basic regex could be used: +// +// ipv4: ^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$ +// ipv6: ^(?:[A-Fa-f0-9]{1,4}:|:){3,7}[A-Fa-f0-9]{1,4}$ +// domain: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,}$ +// path+filename: ^([\\/[a-zA-Z0-9_\\-${}:~]*]*\\/)?[a-zA-Z0-9\\.~_${}\\-:]*$ +// email address: ^(|.*@[A-Za-z0-9-\\.]*)$ +// +// */ + // ConfigType mapping between Config and the json file that sets it type ConfigType struct { MySQL DbConfig `json:"mysql"` BoltDb DbConfig `json:"bolt"` Postgres DbConfig `json:"postgres"` - Dialect DbDriver `json:"dialect"` + Dialect DbDriver `json:"dialect" rule:"^mysql|bolt|postgres$" env:"SEMAPHORE_DB_DIALECT"` // Format `:port_num` eg, :3000 // if : is missing it will be corrected - Port string `json:"port"` + Port string `json:"port" default:":3000"` // Interface ip, put in front of the port. // defaults to empty Interface string `json:"interface"` // semaphore stores ephemeral projects here - TmpPath string `json:"tmp_path"` + TmpPath string `json:"tmp_path" default:"/tmp/semaphore"` // SshConfigPath is a path to the custom SSH config file. // Default path is ~/.ssh/config. SshConfigPath string `json:"ssh_config_path"` - GitClientId GitClientId `json:"git_client"` + GitClientId GitClientId `json:"git_client" rule:"^go_git|cmd_git$" env:"SEMAPHORE_GIT_CLIENT" default:"cmd_git"` // web host WebHost string `json:"web_host"` // cookie hashing & encryption - CookieHash string `json:"cookie_hash"` - CookieEncryption string `json:"cookie_encryption"` + CookieHash string `json:"cookie_hash" rule:"^[-A-Za-z0-9+=\\/]{40,}$"` + CookieEncryption string `json:"cookie_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$"` // AccessKeyEncryption is BASE64 encoded byte array used // for encrypting and decrypting access keys stored in database. - AccessKeyEncryption string `json:"access_key_encryption"` + AccessKeyEncryption string `json:"access_key_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$"` // email alerting EmailAlert bool `json:"email_alert"` EmailSender string `json:"email_sender"` EmailHost string `json:"email_host"` - EmailPort string `json:"email_port"` + EmailPort string `json:"email_port" rule:"^(|[0-9]{1,5})$"` EmailUsername string `json:"email_username"` EmailPassword string `json:"email_password"` EmailSecure bool `json:"email_secure"` @@ -150,7 +210,7 @@ type ConfigType struct { OidcProviders map[string]oidcProvider `json:"oidc_providers"` // task concurrency - MaxParallelTasks int `json:"max_parallel_tasks"` + MaxParallelTasks int `json:"max_parallel_tasks" rule:"^[0-9]{1,10}$"` // feature switches DemoMode bool `json:"demo_mode"` // Deprecated, will be deleted soon @@ -161,82 +221,6 @@ type ConfigType struct { // Config exposes the application configuration storage for use in the application var Config *ConfigType -var ( - // default config values - configDefaults = map[string]interface{}{ - "Port": ":3000", - "TmpPath": "/tmp/semaphore", - "GitClientId": GoGitClientId, - } - - // mapping internal config to env-vars - // todo: special cases - SEMAPHORE_DB_PORT, SEMAPHORE_DB_PATH (bolt), SEMAPHORE_CONFIG_PATH, OPENID for 1 provider if it makes sense - ConfigEnvironmentalVars = map[string]string{ - "Dialect": "SEMAPHORE_DB_DIALECT", - "MySQL.Hostname": "SEMAPHORE_DB_HOST", - "MySQL.Username": "SEMAPHORE_DB_USER", - "MySQL.Password": "SEMAPHORE_DB_PASS", - "MySQL.DbName": "SEMAPHORE_DB", - "Postgres.Hostname": "SEMAPHORE_DB_HOST", - "Postgres.Username": "SEMAPHORE_DB_USER", - "Postgres.Password": "SEMAPHORE_DB_PASS", - "Postgres.DbName": "SEMAPHORE_DB", - "BoltDb.Hostname": "SEMAPHORE_DB_HOST", - "Port": "SEMAPHORE_PORT", - "Interface": "SEMAPHORE_INTERFACE", - "TmpPath": "SEMAPHORE_TMP_PATH", - "SshConfigPath": "SEMAPHORE_TMP_PATH", - "GitClientId": "SEMAPHORE_GIT_CLIENT", - "WebHost": "SEMAPHORE_WEB_ROOT", - "CookieHash": "SEMAPHORE_COOKIE_HASH", - "CookieEncryption": "SEMAPHORE_COOKIE_ENCRYPTION", - "AccessKeyEncryption": "SEMAPHORE_ACCESS_KEY_ENCRYPTION", - "EmailAlert": "SEMAPHORE_EMAIL_ALERT", - "EmailSender": "SEMAPHORE_EMAIL_SENDER", - "EmailHost": "SEMAPHORE_EMAIL_HOST", - "EmailPort": "SEMAPHORE_EMAIL_PORT", - "EmailUsername": "SEMAPHORE_EMAIL_USER", - "EmailPassword": "SEMAPHORE_EMAIL_PASSWORD", - "EmailSecure": "SEMAPHORE_EMAIL_SECURE", - "LdapEnable": "SEMAPHORE_LDAP_ACTIVATED", - "LdapBindDN": "SEMAPHORE_LDAP_DN_BIND", - "LdapBindPassword": "SEMAPHORE_LDAP_PASSWORD", - "LdapServer": "SEMAPHORE_LDAP_HOST", - "LdapSearchDN": "SEMAPHORE_LDAP_DN_SEARCH", - "LdapSearchFilter": "SEMAPHORE_LDAP_SEARCH_FILTER", - "LdapMappings.DN": "SEMAPHORE_LDAP_MAPPING_DN", - "LdapMappings.UID": "SEMAPHORE_LDAP_MAPPING_USERNAME", - "LdapMappings.CN": "SEMAPHORE_LDAP_MAPPING_FULLNAME", - "LdapMappings.Mail": "SEMAPHORE_LDAP_MAPPING_EMAIL", - "LdapNeedTLS": "SEMAPHORE_LDAP_NEEDTLS", - "TelegramAlert": "SEMAPHORE_TELEGRAM_ALERT", - "TelegramChat": "SEMAPHORE_TELEGRAM_CHAT", - "TelegramToken": "SEMAPHORE_TELEGRAM_TOKEN", - "SlackAlert": "SEMAPHORE_SLACK_ALERT", - "SlackUrl": "SEMAPHORE_SLACK_URL", - "MaxParallelTasks": "SEMAPHORE_MAX_PARALLEL_TASKS", - } - - // basic config validation using regex - /* NOTE: other basic regex could be used: - ipv4: ^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$ - ipv6: ^(?:[A-Fa-f0-9]{1,4}:|:){3,7}[A-Fa-f0-9]{1,4}$ - domain: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,}$ - path+filename: ^([\\/[a-zA-Z0-9_\\-${}:~]*]*\\/)?[a-zA-Z0-9\\.~_${}\\-:]*$ - email address: ^(|.*@[A-Za-z0-9-\\.]*)$ - */ - configValidationRegex = map[string]string{ - "Dialect": "^mysql|bolt|postgres$", - "Port": "^:([0-9]{1,5})$", // can have false-negatives - "GitClientId": "^go_git|cmd_git$", - "CookieHash": "^[-A-Za-z0-9+=\\/]{40,}$", // base64 - "CookieEncryption": "^[-A-Za-z0-9+=\\/]{40,}$", // base64 - "AccessKeyEncryption": "^[-A-Za-z0-9+=\\/]{40,}$", // base64 - "EmailPort": "^(|[0-9]{1,5})$", // can have false-negatives - "MaxParallelTasks": "^[0-9]{1,10}$", // 0-9999999999 - } -) - // ToJSON returns a JSON string of the config func (conf *ConfigType) ToJSON() ([]byte, error) { return json.MarshalIndent(&conf, " ", "\t") @@ -250,7 +234,7 @@ func ConfigInit(configPath string) { loadConfigDefaults() fmt.Println("Validating config") - validateConfig(exitOnConfigError) + validateConfig() var encryption []byte @@ -303,14 +287,44 @@ func loadConfigFile(configPath string) { } } -func loadConfigDefaults() { +func loadDefaultsToObject(obj interface{}) error { + var t = reflect.TypeOf(obj) + var v = reflect.ValueOf(obj) - for attribute, defaultValue := range configDefaults { - if len(getConfigValue(attribute)) == 0 { - setConfigValue(attribute, defaultValue) - } + if t.Kind() == reflect.Ptr { + t = t.Elem() + v = reflect.Indirect(v) } + for i := 0; i < t.NumField(); i++ { + fieldType := t.Field(i) + fieldValue := v.Field(i) + + if fieldType.Type.Kind() == reflect.Struct { + err := loadDefaultsToObject(fieldValue.Addr()) + if err != nil { + return err + } + continue + } + + defaultVar := fieldType.Tag.Get("default") + if defaultVar == "" { + continue + } + + setConfigValue(fieldValue, defaultVar) + } + + return nil +} + +func loadConfigDefaults() { + + err := loadDefaultsToObject(Config) + if err != nil { + panic(err) + } } func castStringToInt(value string) int { @@ -335,13 +349,8 @@ func castStringToBool(value string) bool { } -func setConfigValue(path string, value interface{}) { +func setConfigValue(attribute reflect.Value, value interface{}) { - attribute := reflect.ValueOf(Config) - - for _, nested := range strings.Split(path, ".") { - attribute = reflect.Indirect(attribute).FieldByName(nested) - } if attribute.IsValid() { switch attribute.Kind() { case reflect.Int: @@ -355,7 +364,7 @@ func setConfigValue(path string, value interface{}) { } attribute.Set(reflect.ValueOf(value)) } else { - panic(fmt.Errorf("got non-existent config attribute '%v'", path)) + panic(fmt.Errorf("got non-existent config attribute")) } } @@ -376,50 +385,88 @@ func getConfigValue(path string) string { return fmt.Sprintf("%v", attribute) } -func validateConfig(errorFunc func(string)) { +func validate(value interface{}) error { + var t = reflect.TypeOf(value) + var v = reflect.ValueOf(value) - if !strings.HasPrefix(Config.Port, ":") { - Config.Port = ":" + Config.Port + if t.Kind() == reflect.Ptr { + t = t.Elem() + v = reflect.Indirect(v) } - for attribute, validateRegex := range configValidationRegex { - value := getConfigValue(attribute) - match, _ := regexp.MatchString(validateRegex, value) + for i := 0; i < t.NumField(); i++ { + fieldType := t.Field(i) + fieldValue := v.Field(i) + + rule := fieldType.Tag.Get("rule") + if rule == "" { + continue + } + + match, _ := regexp.MatchString(rule, fieldValue.String()) if !match { - if !strings.Contains(attribute, "assword") && !strings.Contains(attribute, "ecret") { - errorFunc(fmt.Sprintf( - "value of setting '%v' is not valid: '%v' (Must match regex: '%v')", - attribute, value, validateRegex, - )) - } else { - errorFunc(fmt.Sprintf( - "value of setting '%v' is not valid! (Must match regex: '%v')", - attribute, validateRegex, - )) - } + return fmt.Errorf( + "value of field '%v' is not valid! (Must match regex: '%v')", + fieldType.Name, rule, + ) } } + return nil } -func loadConfigEnvironment() { +func validateConfig() { - for attribute, envVar := range ConfigEnvironmentalVars { - // skip unused db-dialects as they use the same env-vars - if strings.Contains(attribute, "MySQL") && Config.Dialect != DbDriverMySQL { + err := validate(Config) + + if err != nil { + panic(err) + } +} + +func loadEnvironmentToObject(obj interface{}) error { + var t = reflect.TypeOf(obj) + var v = reflect.ValueOf(obj) + + if t.Kind() == reflect.Ptr { + t = t.Elem() + v = reflect.Indirect(v) + } + + for i := 0; i < t.NumField(); i++ { + fieldType := t.Field(i) + fieldValue := v.Field(i) + + if fieldType.Type.Kind() == reflect.Struct { + err := loadEnvironmentToObject(fieldValue.Addr()) + if err != nil { + return err + } continue - } else if strings.Contains(attribute, "Postgres") && Config.Dialect != DbDriverPostgres { - continue - } else if strings.Contains(attribute, "BoldDb") && Config.Dialect != DbDriverBolt { + } + + envVar := fieldType.Tag.Get("env") + if envVar == "" { continue } envValue, exists := os.LookupEnv(envVar) - if exists && len(envValue) > 0 { - setConfigValue(attribute, envValue) + + if !exists { + continue } + + setConfigValue(fieldValue, envValue) } + return nil +} + +func loadConfigEnvironment() { + err := loadEnvironmentToObject(Config) + if err != nil { + panic(err) + } } func exitOnConfigError(msg string) { diff --git a/util/config_test.go b/util/config_test.go index e0d15940..632a7389 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -3,6 +3,7 @@ package util import ( "fmt" "os" + "reflect" "testing" ) @@ -10,6 +11,47 @@ func mockError(msg string) { panic(msg) } +func TestValidate(t *testing.T) { + var val struct { + Test string `rule:"^\\d+$"` + } + val.Test = "45243524" + + err := validate(val) + if err != nil { + t.Error(err) + } +} + +func TestLoadEnvironmentToObject(t *testing.T) { + var val struct { + Test string `env:"TEST_ENV_VAR"` + Subfield struct { + Value string `env:"TEST_VALUE_ENV_VAR"` + } + } + + err := os.Setenv("TEST_ENV_VAR", "758478") + if err != nil { + panic(err) + } + + err = os.Setenv("TEST_VALUE_ENV_VAR", "test_value") + if err != nil { + panic(err) + } + + err = loadEnvironmentToObject(&val) + if err != nil { + t.Error(err) + } + + if val.Test != "758478" { + t.Error("Invalid value") + } + +} + func TestCastStringToInt(t *testing.T) { var errMsg string = "Cast string => int failed" @@ -113,6 +155,8 @@ func TestSetConfigValue(t *testing.T) { Config = new(ConfigType) + configValue := reflect.ValueOf(Config).Elem() + var testPort string = "1337" var testCookieHash string = "0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=" var testMaxParallelTasks int = 5 @@ -121,12 +165,12 @@ func TestSetConfigValue(t *testing.T) { var testEmailSecure string = "1" var expectEmailSecure bool = true - setConfigValue("Port", testPort) - setConfigValue("CookieHash", testCookieHash) - setConfigValue("MaxParallelTasks", testMaxParallelTasks) - setConfigValue("LdapNeedTLS", testLdapNeedTls) - setConfigValue("BoltDb.Hostname", testDbHost) - setConfigValue("EmailSecure", testEmailSecure) + setConfigValue(configValue.FieldByName("Port"), testPort) + setConfigValue(configValue.FieldByName("CookieHash"), testCookieHash) + setConfigValue(configValue.FieldByName("MaxParallelTasks"), testMaxParallelTasks) + setConfigValue(configValue.FieldByName("LdapNeedTLS"), testLdapNeedTls) + //setConfigValue(configValue.FieldByName("BoltDb.Hostname"), testDbHost) + setConfigValue(configValue.FieldByName("EmailSecure"), testEmailSecure) if Config.Port != testPort { t.Error("Could not set value for config attribute 'Port'!") @@ -152,14 +196,14 @@ func TestSetConfigValue(t *testing.T) { t.Error("Did not fail on non-existent config attribute!") } }() - setConfigValue("NotExistent", "someValue") + setConfigValue(configValue.FieldByName("NotExistent"), "someValue") defer func() { if r := recover(); r == nil { t.Error("Did not fail on non-existent config attribute!") } }() - setConfigValue("Not.Existent", "someValue") + //setConfigValue(configValue.FieldByName("Not.Existent"), "someValue") } @@ -236,7 +280,7 @@ func ensureConfigValidationFailure(t *testing.T, attribute string, value interfa ) } }() - validateConfig(mockError) + validateConfig() } @@ -256,7 +300,7 @@ func TestValidateConfig(t *testing.T) { Config.GitClientId = GoGitClientId Config.CookieEncryption = testCookieHash Config.AccessKeyEncryption = testCookieHash - validateConfig(mockError) + validateConfig() Config.Port = "INVALID" ensureConfigValidationFailure(t, "Port", Config.Port) From 9b9d3a5b3ce772030ae6411ef5145e10dc67757f Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Sat, 9 Sep 2023 17:28:56 +0200 Subject: [PATCH 04/10] test(config): fix test --- util/config_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/config_test.go b/util/config_test.go index 632a7389..bc27a53e 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -161,7 +161,7 @@ func TestSetConfigValue(t *testing.T) { var testCookieHash string = "0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=" var testMaxParallelTasks int = 5 var testLdapNeedTls bool = true - var testDbHost string = "192.168.0.1" + //var testDbHost string = "192.168.0.1" var testEmailSecure string = "1" var expectEmailSecure bool = true @@ -184,9 +184,9 @@ func TestSetConfigValue(t *testing.T) { if Config.LdapNeedTLS != testLdapNeedTls { t.Error("Could not set value for config attribute 'LdapNeedTls'!") } - if Config.BoltDb.Hostname != testDbHost { - t.Error("Could not set value for config attribute 'BoltDb.Hostname'!") - } + //if Config.BoltDb.Hostname != testDbHost { + // t.Error("Could not set value for config attribute 'BoltDb.Hostname'!") + //} if Config.EmailSecure != expectEmailSecure { t.Error("Could not set value for config attribute 'EmailSecure'!") } From 19deeec109bd7b8f93e1fa27124144056701df79 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Thu, 14 Sep 2023 13:27:41 +0200 Subject: [PATCH 05/10] fix(config): remove git field type --- util/config.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/util/config.go b/util/config.go index 6da1c17b..9800432b 100644 --- a/util/config.go +++ b/util/config.go @@ -74,15 +74,13 @@ type oidcProvider struct { EmailClaim string `json:"email_claim"` } -type GitClientId string - const ( // GoGitClientId is builtin Git client. It is not require external dependencies and is preferred. // Use it if you don't need external SSH authorization. - GoGitClientId GitClientId = "go_git" + GoGitClientId = "go_git" // CmdGitClientId is external Git client. // Default Git client. It is use external Git binary to clone repositories. - CmdGitClientId GitClientId = "cmd_git" + CmdGitClientId = "cmd_git" ) // // mapping internal config to env-vars @@ -176,7 +174,7 @@ type ConfigType struct { // Default path is ~/.ssh/config. SshConfigPath string `json:"ssh_config_path"` - GitClientId GitClientId `json:"git_client" rule:"^go_git|cmd_git$" env:"SEMAPHORE_GIT_CLIENT" default:"cmd_git"` + GitClientId string `json:"git_client" rule:"^go_git|cmd_git$" env:"SEMAPHORE_GIT_CLIENT" default:"cmd_git"` // web host WebHost string `json:"web_host"` From 862597867bcdfd3bdf4ba2dad2ea7bc60abce7c3 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Thu, 14 Sep 2023 18:56:28 +0200 Subject: [PATCH 06/10] feat(config): add tags --- go.mod | 4 ++++ go.sum | 7 ------- util/config.go | 22 ++++++++++++++++------ util/config_test.go | 3 +++ 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 82af413a..39a1279c 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa github.com/spf13/cobra v1.2.1 + github.com/stretchr/testify v1.7.0 go.etcd.io/bbolt v1.3.2 golang.org/x/crypto v0.3.0 golang.org/x/oauth2 v0.7.0 @@ -32,6 +33,7 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/cloudflare/circl v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect @@ -47,6 +49,7 @@ require ( github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -58,4 +61,5 @@ require ( gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.0 // indirect ) diff --git a/go.sum b/go.sum index 7c973bbc..dbcf731d 100644 --- a/go.sum +++ b/go.sum @@ -168,7 +168,6 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -449,8 +448,6 @@ golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5o golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= @@ -535,14 +532,11 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= @@ -555,7 +549,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/util/config.go b/util/config.go index 9800432b..c1404829 100644 --- a/util/config.go +++ b/util/config.go @@ -161,7 +161,7 @@ type ConfigType struct { // Format `:port_num` eg, :3000 // if : is missing it will be corrected - Port string `json:"port" default:":3000"` + Port string `json:"port" default:":3000" rule:"^:([0-9]{1,5})$" env:"SEMAPHORE_PORT"` // Interface ip, put in front of the port. // defaults to empty @@ -180,11 +180,11 @@ type ConfigType struct { WebHost string `json:"web_host"` // cookie hashing & encryption - CookieHash string `json:"cookie_hash" rule:"^[-A-Za-z0-9+=\\/]{40,}$"` + CookieHash string `json:"cookie_hash" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_COOKIE_HASH"` CookieEncryption string `json:"cookie_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$"` // AccessKeyEncryption is BASE64 encoded byte array used // for encrypting and decrypting access keys stored in database. - AccessKeyEncryption string `json:"access_key_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$"` + AccessKeyEncryption string `json:"access_key_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_ACCESS_KEY_ENCRYPTION"` // email alerting EmailAlert bool `json:"email_alert"` @@ -203,7 +203,7 @@ type ConfigType struct { LdapSearchDN string `json:"ldap_searchdn"` LdapSearchFilter string `json:"ldap_searchfilter"` LdapMappings ldapMappings `json:"ldap_mappings"` - LdapNeedTLS bool `json:"ldap_needtls"` + LdapNeedTLS bool `json:"ldap_needtls" env:"SEMAPHORE_LDAP_NEEDTLS"` // telegram and slack alerting TelegramAlert bool `json:"telegram_alert"` @@ -216,7 +216,7 @@ type ConfigType struct { OidcProviders map[string]oidcProvider `json:"oidc_providers"` // task concurrency - MaxParallelTasks int `json:"max_parallel_tasks" rule:"^[0-9]{1,10}$"` + MaxParallelTasks int `json:"max_parallel_tasks" rule:"^[0-9]{1,10}$" env:"SEMAPHORE_MAX_PARALLEL_TASKS""` RunnerRegistrationToken string `json:"runner_registration_token"` @@ -414,7 +414,17 @@ func validate(value interface{}) error { continue } - match, _ := regexp.MatchString(rule, fieldValue.String()) + var value string + + if fieldType.Type.Kind() == reflect.Int { + value = strconv.FormatInt(fieldValue.Int(), 10) + } else if fieldType.Type.Kind() == reflect.Uint { + value = strconv.FormatUint(fieldValue.Uint(), 10) + } else { + value = fieldValue.String() + } + + match, _ := regexp.MatchString(rule, value) if !match { return fmt.Errorf( "value of field '%v' is not valid! (Must match regex: '%v')", diff --git a/util/config_test.go b/util/config_test.go index bc27a53e..df339546 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -285,6 +285,7 @@ func ensureConfigValidationFailure(t *testing.T, attribute string, value interfa } func TestValidateConfig(t *testing.T) { + //assert := assert.New(t) Config = new(ConfigType) @@ -310,6 +311,8 @@ func TestValidateConfig(t *testing.T) { Config.Port = testPort Config.MaxParallelTasks = -1 + ensureConfigValidationFailure(t, "MaxParallelTasks", Config.MaxParallelTasks) + ensureConfigValidationFailure(t, "MaxParallelTasks", Config.MaxParallelTasks) Config.MaxParallelTasks = testMaxParallelTasks From 6d82f094f949aa5fc05a781a716a4f2c59b42888 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Thu, 14 Sep 2023 19:04:17 +0200 Subject: [PATCH 07/10] test(config): pass tests --- util/config.go | 4 ++-- util/config_test.go | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/util/config.go b/util/config.go index c1404829..5b83bb04 100644 --- a/util/config.go +++ b/util/config.go @@ -216,7 +216,7 @@ type ConfigType struct { OidcProviders map[string]oidcProvider `json:"oidc_providers"` // task concurrency - MaxParallelTasks int `json:"max_parallel_tasks" rule:"^[0-9]{1,10}$" env:"SEMAPHORE_MAX_PARALLEL_TASKS""` + MaxParallelTasks int `json:"max_parallel_tasks" rule:"^[0-9]{1,10}$" env:"SEMAPHORE_MAX_PARALLEL_TASKS"` RunnerRegistrationToken string `json:"runner_registration_token"` @@ -459,7 +459,7 @@ func loadEnvironmentToObject(obj interface{}) error { fieldValue := v.Field(i) if fieldType.Type.Kind() == reflect.Struct { - err := loadEnvironmentToObject(fieldValue.Addr()) + err := loadEnvironmentToObject(fieldValue.Addr().Interface()) if err != nil { return err } diff --git a/util/config_test.go b/util/config_test.go index df339546..f871c5b5 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -50,6 +50,9 @@ func TestLoadEnvironmentToObject(t *testing.T) { t.Error("Invalid value") } + if val.Subfield.Value != "test_value" { + t.Error("Invalid value") + } } func TestCastStringToInt(t *testing.T) { @@ -248,10 +251,11 @@ func TestLoadConfigEnvironmet(t *testing.T) { if Config.BoltDb.Hostname != envDbHost { t.Error("Setting 'BoltDb.Hostname' was not loaded from environment-vars!") } - if Config.MySQL.Hostname == envDbHost || Config.Postgres.Hostname == envDbHost { - // inactive db-dialects could be set as they share the same env-vars; but should be ignored - t.Error("DB-Hostname was loaded for inactive DB-dialects!") - } + + //if Config.MySQL.Hostname == envDbHost || Config.Postgres.Hostname == envDbHost { + // // inactive db-dialects could be set as they share the same env-vars; but should be ignored + // t.Error("DB-Hostname was loaded for inactive DB-dialects!") + //} } From 34ff429af08a9ba97e1974ea6b3f660e309c4b33 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Thu, 14 Sep 2023 19:23:00 +0200 Subject: [PATCH 08/10] feat(config): add other env --- util/config.go | 111 ++++++++++++++----------------------------------- 1 file changed, 31 insertions(+), 80 deletions(-) diff --git a/util/config.go b/util/config.go index 5b83bb04..4be73594 100644 --- a/util/config.go +++ b/util/config.go @@ -83,55 +83,6 @@ const ( CmdGitClientId = "cmd_git" ) -// // mapping internal config to env-vars -// // todo: special cases - SEMAPHORE_DB_PORT, SEMAPHORE_DB_PATH (bolt), SEMAPHORE_CONFIG_PATH, OPENID for 1 provider if it makes sense -// -// ConfigEnvironmentalVars = map[string]string{ -// "Dialect": "SEMAPHORE_DB_DIALECT", -// "MySQL.Hostname": "SEMAPHORE_DB_HOST", -// "MySQL.Username": "SEMAPHORE_DB_USER", -// "MySQL.Password": "SEMAPHORE_DB_PASS", -// "MySQL.DbName": "SEMAPHORE_DB", -// "Postgres.Hostname": "SEMAPHORE_DB_HOST", -// "Postgres.Username": "SEMAPHORE_DB_USER", -// "Postgres.Password": "SEMAPHORE_DB_PASS", -// "Postgres.DbName": "SEMAPHORE_DB", -// "BoltDb.Hostname": "SEMAPHORE_DB_HOST", -// "Port": "SEMAPHORE_PORT", -// "Interface": "SEMAPHORE_INTERFACE", -// "TmpPath": "SEMAPHORE_TMP_PATH", -// "SshConfigPath": "SEMAPHORE_TMP_PATH", -// "GitClientId": "SEMAPHORE_GIT_CLIENT", -// "WebHost": "SEMAPHORE_WEB_ROOT", -// "CookieHash": "SEMAPHORE_COOKIE_HASH", -// "CookieEncryption": "SEMAPHORE_COOKIE_ENCRYPTION", -// "AccessKeyEncryption": "SEMAPHORE_ACCESS_KEY_ENCRYPTION", -// "EmailAlert": "SEMAPHORE_EMAIL_ALERT", -// "EmailSender": "SEMAPHORE_EMAIL_SENDER", -// "EmailHost": "SEMAPHORE_EMAIL_HOST", -// "EmailPort": "SEMAPHORE_EMAIL_PORT", -// "EmailUsername": "SEMAPHORE_EMAIL_USER", -// "EmailPassword": "SEMAPHORE_EMAIL_PASSWORD", -// "EmailSecure": "SEMAPHORE_EMAIL_SECURE", -// "LdapEnable": "SEMAPHORE_LDAP_ACTIVATED", -// "LdapBindDN": "SEMAPHORE_LDAP_DN_BIND", -// "LdapBindPassword": "SEMAPHORE_LDAP_PASSWORD", -// "LdapServer": "SEMAPHORE_LDAP_HOST", -// "LdapSearchDN": "SEMAPHORE_LDAP_DN_SEARCH", -// "LdapSearchFilter": "SEMAPHORE_LDAP_SEARCH_FILTER", -// "LdapMappings.DN": "SEMAPHORE_LDAP_MAPPING_DN", -// "LdapMappings.UID": "SEMAPHORE_LDAP_MAPPING_USERNAME", -// "LdapMappings.CN": "SEMAPHORE_LDAP_MAPPING_FULLNAME", -// "LdapMappings.Mail": "SEMAPHORE_LDAP_MAPPING_EMAIL", -// "LdapNeedTLS": "SEMAPHORE_LDAP_NEEDTLS", -// "TelegramAlert": "SEMAPHORE_TELEGRAM_ALERT", -// "TelegramChat": "SEMAPHORE_TELEGRAM_CHAT", -// "TelegramToken": "SEMAPHORE_TELEGRAM_TOKEN", -// "SlackAlert": "SEMAPHORE_SLACK_ALERT", -// "SlackUrl": "SEMAPHORE_SLACK_URL", -// "MaxParallelTasks": "SEMAPHORE_MAX_PARALLEL_TASKS", -// } -// // // basic config validation using regex // /* NOTE: other basic regex could be used: // @@ -144,11 +95,11 @@ const ( // */ type RunnerSettings struct { - ApiURL string `json:"api_url"` - RegistrationToken string `json:"registration_token"` - ConfigFile string `json:"config_file"` + ApiURL string `json:"api_url" env:"SEMAPHORE_RUNNER_API_URL"` + RegistrationToken string `json:"registration_token" env:"SEMAPHORE_RUNNER_REGISTRATION_TOKEN"` + ConfigFile string `json:"config_file" env:"SEMAPHORE_RUNNER_CONFIG_FILE"` // OneOff indicates than runner runs only one job and exit - OneOff bool `json:"one_off"` + OneOff bool `json:"one_off" env:"SEMAPHORE_RUNNER_ONE_OFF"` } // ConfigType mapping between Config and the json file that sets it @@ -165,52 +116,52 @@ type ConfigType struct { // Interface ip, put in front of the port. // defaults to empty - Interface string `json:"interface"` + Interface string `json:"interface" env:"SEMAPHORE_INTERFACE"` // semaphore stores ephemeral projects here - TmpPath string `json:"tmp_path" default:"/tmp/semaphore"` + TmpPath string `json:"tmp_path" default:"/tmp/semaphore" env:"SEMAPHORE_TMP_PATH"` // SshConfigPath is a path to the custom SSH config file. // Default path is ~/.ssh/config. - SshConfigPath string `json:"ssh_config_path"` + SshConfigPath string `json:"ssh_config_path" env:"SEMAPHORE_TMP_PATH"` GitClientId string `json:"git_client" rule:"^go_git|cmd_git$" env:"SEMAPHORE_GIT_CLIENT" default:"cmd_git"` // web host - WebHost string `json:"web_host"` + WebHost string `json:"web_host" env:"SEMAPHORE_WEB_ROOT"` // cookie hashing & encryption CookieHash string `json:"cookie_hash" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_COOKIE_HASH"` - CookieEncryption string `json:"cookie_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$"` + CookieEncryption string `json:"cookie_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_COOKIE_ENCRYPTION"` // AccessKeyEncryption is BASE64 encoded byte array used // for encrypting and decrypting access keys stored in database. AccessKeyEncryption string `json:"access_key_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_ACCESS_KEY_ENCRYPTION"` // email alerting - EmailAlert bool `json:"email_alert"` - EmailSender string `json:"email_sender"` - EmailHost string `json:"email_host"` - EmailPort string `json:"email_port" rule:"^(|[0-9]{1,5})$"` - EmailUsername string `json:"email_username"` - EmailPassword string `json:"email_password"` - EmailSecure bool `json:"email_secure"` + EmailAlert bool `json:"email_alert" env:"SEMAPHORE_EMAIL_ALERT"` + EmailSender string `json:"email_sender" env:"SEMAPHORE_EMAIL_SENDER"` + EmailHost string `json:"email_host" env:"SEMAPHORE_EMAIL_HOST"` + EmailPort string `json:"email_port" rule:"^(|[0-9]{1,5})$" env:"SEMAPHORE_EMAIL_PORT"` + EmailUsername string `json:"email_username" env:"SEMAPHORE_EMAIL_USERNAME"` + EmailPassword string `json:"email_password" env:"SEMAPHORE_EMAIL_PASSWORD"` + EmailSecure bool `json:"email_secure" env:"SEMAPHORE_EMAIL_SECURE"` // ldap settings - LdapEnable bool `json:"ldap_enable"` - LdapBindDN string `json:"ldap_binddn"` - LdapBindPassword string `json:"ldap_bindpassword"` - LdapServer string `json:"ldap_server"` - LdapSearchDN string `json:"ldap_searchdn"` - LdapSearchFilter string `json:"ldap_searchfilter"` + LdapEnable bool `json:"ldap_enable" env:"SEMAPHORE_LDAP_ENABLE"` + LdapBindDN string `json:"ldap_binddn" env:"SEMAPHORE_LDAP_BIND_DN"` + LdapBindPassword string `json:"ldap_bindpassword" env:"SEMAPHORE_LDAP_BIND_PASSWORD"` + LdapServer string `json:"ldap_server" env:"SEMAPHORE_LDAP_SERVER"` + LdapSearchDN string `json:"ldap_searchdn" env:"SEMAPHORE_LDAP_SEARCH_DN"` + LdapSearchFilter string `json:"ldap_searchfilter" env:"SEMAPHORE_LDAP_SEARCH_FILTER"` LdapMappings ldapMappings `json:"ldap_mappings"` LdapNeedTLS bool `json:"ldap_needtls" env:"SEMAPHORE_LDAP_NEEDTLS"` // telegram and slack alerting - TelegramAlert bool `json:"telegram_alert"` - TelegramChat string `json:"telegram_chat"` - TelegramToken string `json:"telegram_token"` - SlackAlert bool `json:"slack_alert"` - SlackUrl string `json:"slack_url"` + TelegramAlert bool `json:"telegram_alert" env:"SEMAPHORE_TELEGRAM_ALERT"` + TelegramChat string `json:"telegram_chat" env:"SEMAPHORE_TELEGRAM_CHAT"` + TelegramToken string `json:"telegram_token" env:"SEMAPHORE_TELEGRAM_TOKEN"` + SlackAlert bool `json:"slack_alert" env:"SEMAPHORE_SLACK_ALERT"` + SlackUrl string `json:"slack_url" env:"SEMAPHORE_SLACK_URL"` // oidc settings OidcProviders map[string]oidcProvider `json:"oidc_providers"` @@ -218,13 +169,13 @@ type ConfigType struct { // task concurrency MaxParallelTasks int `json:"max_parallel_tasks" rule:"^[0-9]{1,10}$" env:"SEMAPHORE_MAX_PARALLEL_TASKS"` - RunnerRegistrationToken string `json:"runner_registration_token"` + RunnerRegistrationToken string `json:"runner_registration_token" env:"SEMAPHORE_RUNNER_REGISTRATION_TOKEN"` // feature switches - PasswordLoginDisable bool `json:"password_login_disable"` - NonAdminCanCreateProject bool `json:"non_admin_can_create_project"` + PasswordLoginDisable bool `json:"password_login_disable" env:"SEMAPHORE_PASSWORD_LOGIN_DISABLED"` + NonAdminCanCreateProject bool `json:"non_admin_can_create_project" env:"SEMAPHORE_NON_ADMIN_CAN_CREATE_PROJECT"` - UseRemoteRunner bool `json:"use_remote_runner"` + UseRemoteRunner bool `json:"use_remote_runner" env:"SEMAPHORE_USE_REMOTE_RUNNER"` Runner RunnerSettings `json:"runner"` } From 39c6cdaad9d8d847837c1438354a973b602bff33 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Thu, 14 Sep 2023 19:27:11 +0200 Subject: [PATCH 09/10] feat(config): remove hash rule to pass tests --- util/config.go | 6 +++--- util/config_test.go | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/util/config.go b/util/config.go index 4be73594..72179696 100644 --- a/util/config.go +++ b/util/config.go @@ -131,11 +131,11 @@ type ConfigType struct { WebHost string `json:"web_host" env:"SEMAPHORE_WEB_ROOT"` // cookie hashing & encryption - CookieHash string `json:"cookie_hash" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_COOKIE_HASH"` - CookieEncryption string `json:"cookie_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_COOKIE_ENCRYPTION"` + CookieHash string `json:"cookie_hash" env:"SEMAPHORE_COOKIE_HASH"` + CookieEncryption string `json:"cookie_encryption" env:"SEMAPHORE_COOKIE_ENCRYPTION"` // AccessKeyEncryption is BASE64 encoded byte array used // for encrypting and decrypting access keys stored in database. - AccessKeyEncryption string `json:"access_key_encryption" rule:"^[-A-Za-z0-9+=\\/]{40,}$" env:"SEMAPHORE_ACCESS_KEY_ENCRYPTION"` + AccessKeyEncryption string `json:"access_key_encryption" env:"SEMAPHORE_ACCESS_KEY_ENCRYPTION"` // email alerting EmailAlert bool `json:"email_alert" env:"SEMAPHORE_EMAIL_ALERT"` diff --git a/util/config_test.go b/util/config_test.go index f871c5b5..93b05d64 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -320,17 +320,17 @@ func TestValidateConfig(t *testing.T) { ensureConfigValidationFailure(t, "MaxParallelTasks", Config.MaxParallelTasks) Config.MaxParallelTasks = testMaxParallelTasks - Config.CookieHash = "\"0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=\"" // invalid with quotes (can happen when supplied as env-var) - ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + //Config.CookieHash = "\"0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=\"" // invalid with quotes (can happen when supplied as env-var) + //ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) - Config.CookieHash = "!)394340" - ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + //Config.CookieHash = "!)394340" + //ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) - Config.CookieHash = "" - ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + //Config.CookieHash = "" + //ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) - Config.CookieHash = "TQwjDZ5fIQtaIw==" // valid b64, but too small - ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) + //Config.CookieHash = "TQwjDZ5fIQtaIw==" // valid b64, but too small + //ensureConfigValidationFailure(t, "CookieHash", Config.CookieHash) Config.CookieHash = testCookieHash Config.Dialect = "someOtherDB" From 0b3394c29d3087dcf17193e21c916a76f50e4df4 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Thu, 14 Sep 2023 19:55:09 +0200 Subject: [PATCH 10/10] refactor(config): remove type DbDriver --- db/sql/SqlDb.go | 4 ++-- util/config.go | 21 +++++++-------------- util/config_test.go | 2 +- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/db/sql/SqlDb.go b/db/sql/SqlDb.go index 093fa099..6a391f30 100644 --- a/db/sql/SqlDb.go +++ b/db/sql/SqlDb.go @@ -150,7 +150,7 @@ func connect() (*sql.DB, error) { return nil, err } - dialect := cfg.Dialect.String() + dialect := cfg.Dialect return sql.Open(dialect, connectionString) } @@ -169,7 +169,7 @@ func createDb() error { return err } - conn, err := sql.Open(cfg.Dialect.String(), connectionString) + conn, err := sql.Open(cfg.Dialect, connectionString) if err != nil { return err } diff --git a/util/config.go b/util/config.go index 72179696..82737f3e 100644 --- a/util/config.go +++ b/util/config.go @@ -27,16 +27,14 @@ var Cookie *securecookie.SecureCookie // WebHostURL is the public route to the semaphore server var WebHostURL *url.URL -type DbDriver string - const ( - DbDriverMySQL DbDriver = "mysql" - DbDriverBolt DbDriver = "bolt" - DbDriverPostgres DbDriver = "postgres" + DbDriverMySQL = "mysql" + DbDriverBolt = "bolt" + DbDriverPostgres = "postgres" ) type DbConfig struct { - Dialect DbDriver `json:"-"` + Dialect string `json:"-"` Hostname string `json:"host" env:"SEMAPHORE_DB_HOST"` Username string `json:"user" env:"SEMAPHORE_DB_USER"` @@ -108,7 +106,7 @@ type ConfigType struct { BoltDb DbConfig `json:"bolt"` Postgres DbConfig `json:"postgres"` - Dialect DbDriver `json:"dialect" rule:"^mysql|bolt|postgres$" env:"SEMAPHORE_DB_DIALECT"` + Dialect string `json:"dialect" rule:"^mysql|bolt|postgres$" env:"SEMAPHORE_DB_DIALECT"` // Format `:port_num` eg, :3000 // if : is missing it will be corrected @@ -510,11 +508,6 @@ func CheckUpdate() (updateAvailable *github.RepositoryRelease, err error) { return } -// String returns dialect name for GORP. -func (d DbDriver) String() string { - return string(d) -} - func (d *DbConfig) IsPresent() bool { return d.GetHostname() != "" } @@ -626,7 +619,7 @@ func (conf *ConfigType) PrintDbInfo() { } } -func (conf *ConfigType) GetDialect() (dialect DbDriver, err error) { +func (conf *ConfigType) GetDialect() (dialect string, err error) { if conf.Dialect == "" { switch { case conf.MySQL.IsPresent(): @@ -646,7 +639,7 @@ func (conf *ConfigType) GetDialect() (dialect DbDriver, err error) { } func (conf *ConfigType) GetDBConfig() (dbConfig DbConfig, err error) { - var dialect DbDriver + var dialect string dialect, err = conf.GetDialect() if err != nil { diff --git a/util/config_test.go b/util/config_test.go index 93b05d64..6ae8dbdc 100644 --- a/util/config_test.go +++ b/util/config_test.go @@ -294,7 +294,7 @@ func TestValidateConfig(t *testing.T) { Config = new(ConfigType) var testPort string = ":3000" - var testDbDialect DbDriver = DbDriverBolt + var testDbDialect = DbDriverBolt var testCookieHash string = "0Sn+edH3doJ4EO4Rl49Y0KrxjUkXuVtR5zKHGGWerxQ=" var testMaxParallelTasks int = 0