🎉 remove redis dependency

- Remove mandrill
- Remove redis
- Sessions & more transparency into who's logged into your user
- Session is stored in cookies (map of two integers.)
- Encrypted sessions, configurable keys (auto-generated)

README.md

@@ -26,23 +26,6 @@ Please open an issue, tell us you database version & configuration.
 
 ---
 
-```
-PING to redis unsuccessful
-... panic here ...
-```
-
-The program cannot reach your redis instance. Check the configuration and test manually with:
-
-```
-nc <IP> 6379
-PING
-+PONG
-```
-
-if `netcat` returns immediately, the port is not reachable.
-
----
-
 ## [Milestones](https://github.com/ansible-semaphore/semaphore/milestones)
 ## [Releases](https://github.com/ansible-semaphore/semaphore/releases)
 

database/mysql.go

@@ -12,9 +12,17 @@ var Mysql *gorp.DbMap
 
 // Mysql database
 func Connect() error {
-	url := util.Config.MySQL.Username + ":" + util.Config.MySQL.Password + "@tcp(" + util.Config.MySQL.Hostname + ")/?parseTime=true&interpolateParams=true"
+	db, err := connect()
+	if err != nil {
+		return err
+	}
 
-	db, err := sql.Open("mysql", url)
+	if err := db.Ping(); err != nil {
+		if err := createDb(); err != nil {
+			return err
+		}
+
+		db, err = connect()
 		if err != nil {
 			return err
 		}
@@ -22,15 +30,31 @@ func Connect() error {
 		if err := db.Ping(); err != nil {
 			return err
 		}
-
-	if _, err := db.Exec("create database if not exists " + util.Config.MySQL.DbName); err != nil {
-		return err
-	}
-
-	if _, err := db.Exec("use " + util.Config.MySQL.DbName); err != nil {
-		return err
 	}
 
 	Mysql = &gorp.DbMap{Db: db, Dialect: gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}}
 	return nil
 }
+
+func createDb() error {
+	cfg := util.Config.MySQL
+	url := cfg.Username + ":" + cfg.Password + "@tcp(" + cfg.Hostname + ")/?parseTime=true&interpolateParams=true"
+
+	db, err := sql.Open("mysql", url)
+	if err != nil {
+		return err
+	}
+
+	if _, err := db.Exec("create database if not exists " + cfg.DbName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func connect() (*sql.DB, error) {
+	cfg := util.Config.MySQL
+	url := cfg.Username + ":" + cfg.Password + "@tcp(" + cfg.Hostname + ")/" + cfg.DbName + "?parseTime=true&interpolateParams=true"
+
+	return sql.Open("mysql", url)
+}

database/redis.go

@@ -1,27 +0,0 @@
-package database
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/ansible-semaphore/semaphore/util"
-	"gopkg.in/redis.v3"
-)
-
-// Redis pool
-var Redis *redis.Client
-
-func init() {
-	Redis = redis.NewClient(&redis.Options{
-		MaxRetries:  2,
-		DialTimeout: 10 * time.Second,
-		Addr:        util.Config.SessionDb,
-	})
-}
-
-func RedisPing() {
-	if _, err := Redis.Ping().Result(); err != nil {
-		fmt.Println("PING to redis unsuccessful")
-		panic(err)
-	}
-}

database/sql_migrations/v1.5.0.sql

@@ -0,0 +1,12 @@
+CREATE TABLE `session` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user_id` int(11) NOT NULL,
+  `created` datetime NOT NULL,
+  `last_active` datetime NOT NULL,
+  `ip` varchar(15) NOT NULL DEFAULT '',
+  `user_agent` text NOT NULL,
+  `expired` tinyint(1) NOT NULL DEFAULT '0',
+  PRIMARY KEY (`id`),
+  KEY `user_id` (`user_id`),
+  KEY `expired` (`expired`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

docker-compose.yml

@@ -1,11 +1,3 @@
-redis:
-  name: redis:latest
-  image: redis
-  expose:
-    - 6379
-  volumes:
-    - /tmp/redis:/data
-
 mongodb:
   image: mongo:latest
   command: mongod --smallfiles --directoryperdb --noprealloc

main.go

@@ -37,7 +37,6 @@ func main() {
 	fmt.Printf("Semaphore %v\n", util.Version)
 	fmt.Printf("Port %v\n", util.Config.Port)
 	fmt.Printf("MySQL %v@%v %v\n", util.Config.MySQL.Username, util.Config.MySQL.Hostname, util.Config.MySQL.DbName)
-	fmt.Printf("Redis %v\n", util.Config.SessionDb)
 	fmt.Printf("Tmp Path (projects home) %v\n", util.Config.TmpPath)
 
 	if err := database.Connect(); err != nil {
@@ -47,7 +46,6 @@ func main() {
 	models.SetupDBLink()
 
 	defer database.Mysql.Db.Close()
-	database.RedisPing()
 
 	if util.Migration {
 		fmt.Println("\n Running DB Migrations")
@@ -78,16 +76,17 @@ func doSetup() int {
  Hello! You will now be guided through a setup to:
 
  1. Set up configuration for a MySQL/MariaDB database
- 2. Set up redis for session storage
+ 2. Set up a path for your playbooks (auto-created)
- 3. Set up a path for your playbooks (auto-created)
+ 3. Run database Migrations
- 4. Run database Migrations
+ 4. Set up initial seamphore user & password
- 5. Set up initial seamphore user & password
 
 `)
 
 	var b []byte
-	setup := util.ScanSetup()
+	setup := util.NewConfig()
 	for true {
+		setup.Scan()
+
 		var err error
 		b, err = json.MarshalIndent(&setup, " ", "\t")
 		if err != nil {
@@ -104,9 +103,11 @@ func doSetup() int {
 		}
 
 		fmt.Println()
-		setup = util.ScanSetup()
+		setup = util.NewConfig()
 	}
 
+	setup.GenerateCookieSecrets()
+
 	fmt.Printf(" Running: mkdir -p %v..\n", setup.TmpPath)
 	os.MkdirAll(setup.TmpPath, 0755)
 
@@ -124,9 +125,6 @@ func doSetup() int {
 		os.Exit(1)
 	}
 
-	fmt.Println(" Pinging redis..")
-	database.RedisPing()
-
 	fmt.Println("\n Running DB Migrations..")
 	if err := migration.MigrateAll(); err != nil {
 		fmt.Printf("\n Database migrations failed!\n %v\n", err.Error())

migration/versionHistory.go

@@ -60,5 +60,6 @@ func init() {
 		{Major: 1, Minor: 2},
 		{Major: 1, Minor: 3},
 		{Major: 1, Minor: 4},
+		{Major: 1, Minor: 5},
 	}
 }

models/Session.go

@@ -1,29 +1,13 @@
 package models
 
-import (
+import "time"
-	"encoding/json"
-)
 
 type Session struct {
-	ID string `json:"-"`
+	ID         int       `db:"id" json:"id"`
-
+	UserID     int       `db:"user_id" json:"user_id"`
-	UserID *int `json:"user_id"`
+	Created    time.Time `db:"created" json:"created"`
-}
+	LastActive time.Time `db:"last_active" json:"last_active"`
-
+	IP         string    `db:"ip" json:"ip"`
-func (session *Session) Encode() []byte {
+	UserAgent  string    `db:"user_agent" json:"user_agent"`
-	js, err := json.Marshal(session)
+	Expired    bool      `db:"expired" json:"expired"`
-	if err != nil {
-		panic(err)
-	}
-
-	return js
-}
-
-func DecodeSession(ID string, sess string) (Session, error) {
-	var session Session
-	err := json.Unmarshal([]byte(sess), &session)
-
-	session.ID = ID
-
-	return session, err
 }

models/models.go

@@ -13,4 +13,5 @@ func SetupDBLink() {
 	database.Mysql.AddTableWithName(TaskOutput{}, "task__output").SetUniqueTogether("task_id", "time")
 	database.Mysql.AddTableWithName(Template{}, "project__template").SetKeys(true, "id")
 	database.Mysql.AddTableWithName(User{}, "user").SetKeys(true, "id")
+	database.Mysql.AddTableWithName(Session{}, "session").SetKeys(true, "id")
 }

playbooks/playbook.yml

@@ -17,7 +17,6 @@
             - nodejs
             - npm
             - mongodb-server
-            - redis-server
             - ansible
             - runit
       - npm: name={{ item }} global=yes

routes/auth.go

@@ -1,12 +1,8 @@
 package routes
 
 import (
-	"crypto/rand"
+	"database/sql"
-	"encoding/base64"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
@@ -14,72 +10,71 @@ import (
 	"github.com/ansible-semaphore/semaphore/models"
 	"github.com/ansible-semaphore/semaphore/util"
 	"github.com/gin-gonic/gin"
-	"gopkg.in/redis.v3"
 )
 
-func resetSessionExpiry(sessionID string, ttl time.Duration) {
-	var cmd *redis.BoolCmd
-
-	if ttl == 0 {
-		cmd = database.Redis.Persist(sessionID)
-	} else {
-		cmd = database.Redis.Expire(sessionID, ttl)
-	}
-
-	if err := cmd.Err(); err != nil {
-		fmt.Println("Cannot reset session expiry:", err)
-	}
-}
-
 func authentication(c *gin.Context) {
-	var redisKey string
+	var userID int
-	ttl := 7 * 24 * time.Hour
 
 	if authHeader := strings.ToLower(c.Request.Header.Get("authorization")); len(authHeader) > 0 {
-		redisKey = "token-session:" + strings.Replace(authHeader, "bearer ", "", 1)
+		var token models.APIToken
-		ttl = 0
+		if err := database.Mysql.SelectOne(&token, "select * from user__token where id=? and expired=0", strings.Replace(authHeader, "bearer ", "", 1)); err != nil {
+			if err == sql.ErrNoRows {
+				c.AbortWithStatus(403)
+				return
+			}
+
+			panic(err)
+		}
+
+		userID = token.UserID
 	} else {
+		// fetch session from cookie
 		cookie, err := c.Request.Cookie("semaphore")
 		if err != nil {
-			// create cookie
+			c.AbortWithStatus(403)
-			new_cookie := make([]byte, 32)
-			if _, err := io.ReadFull(rand.Reader, new_cookie); err != nil {
-				panic(err)
-			}
-
-			cookie_value := url.QueryEscape(base64.URLEncoding.EncodeToString(new_cookie))
-			cookie = &http.Cookie{Name: "semaphore", Value: cookie_value, Path: "/", HttpOnly: true}
-			http.SetCookie(c.Writer, cookie)
-		}
-
-		redisKey = "session:" + cookie.Value
-	}
-
-	s, err := database.Redis.Get(redisKey).Result()
-	if err == redis.Nil {
-		// create a session
-		temp_session := models.Session{}
-		s = string(temp_session.Encode())
-
-		if err := database.Redis.Set(redisKey, s, 0).Err(); err != nil {
-			panic(err)
-		}
-	} else if err != nil {
-		fmt.Println("Cannot get session from redis:", err)
-		c.AbortWithStatus(500)
-
 			return
 		}
 
-	sess, err := models.DecodeSession(redisKey, s)
+		value := make(map[string]interface{})
-	if err != nil {
+		if err = util.Cookie.Decode("semaphore", cookie.Value, &value); err != nil {
-		fmt.Println("Cannot decode session:", err)
+			c.AbortWithStatus(403)
-		util.AuthFailed(c)
+			panic(err)
+		}
+
+		user, ok := value["user"]
+		sessionVal, okSession := value["session"]
+		if !ok || !okSession {
+			c.AbortWithStatus(403)
 			return
 		}
 
-	if sess.UserID != nil {
+		userID = user.(int)
-		user, err := models.FetchUser(*sess.UserID)
+		sessionID := sessionVal.(int)
+
+		// fetch session
+		var session models.Session
+		if err := database.Mysql.SelectOne(&session, "select * from session where id=? and user_id=? and expired=0", sessionID, userID); err != nil {
+			c.AbortWithStatus(403)
+			return
+		}
+
+		if time.Now().Sub(session.LastActive).Hours() > 7*24 {
+			// more than week old unused session
+			// destroy.
+			if _, err := database.Mysql.Exec("update session set expired=1 where id=?", sessionID); err != nil {
+				panic(err)
+			}
+
+			c.AbortWithStatus(403)
+			return
+		}
+
+		if _, err := database.Mysql.Exec("update session set last_active=NOW() where id=?", sessionID); err != nil {
+			panic(err)
+		}
+	}
+
+	user, err := models.FetchUser(userID)
 	if err != nil {
 		fmt.Println("Can't find user", err)
 		c.AbortWithStatus(403)
@@ -88,19 +83,3 @@ func authentication(c *gin.Context) {
 
 	c.Set("user", user)
 }
-
-	// reset session expiry
-	go resetSessionExpiry(redisKey, ttl)
-
-	c.Set("session", sess)
-	c.Next()
-}
-
-func MustAuthenticate(c *gin.Context) {
-	if _, exists := c.Get("user"); !exists {
-		util.AuthFailed(c)
-		return
-	}
-
-	c.Next()
-}

routes/login.go

@@ -2,16 +2,17 @@ package routes
 
 import (
 	"database/sql"
+	"net/http"
 	"net/mail"
 	"strings"
 	"time"
 
-	"golang.org/x/crypto/bcrypt"
-
 	"github.com/ansible-semaphore/semaphore/database"
 	"github.com/ansible-semaphore/semaphore/models"
+	"github.com/ansible-semaphore/semaphore/util"
 	"github.com/gin-gonic/gin"
 	sq "github.com/masterminds/squirrel"
+	"golang.org/x/crypto/bcrypt"
 )
 
 func login(c *gin.Context) {
@@ -53,22 +54,36 @@ func login(c *gin.Context) {
 		return
 	}
 
-	session := c.MustGet("session").(models.Session)
+	session := models.Session{
-	session.UserID = &user.ID
+		UserID:     user.ID,
-
+		Created:    time.Now(),
-	status := database.Redis.Set(session.ID, string(session.Encode()), 7*24*time.Hour)
+		LastActive: time.Now(),
-	if err := status.Err(); err != nil {
+		IP:         c.ClientIP(),
+		UserAgent:  c.Request.Header.Get("user-agent"),
+		Expired:    false,
+	}
+	if err := database.Mysql.Insert(&session); err != nil {
 		panic(err)
 	}
 
+	encoded, err := util.Cookie.Encode("semaphore", map[string]interface{}{
+		"user":    user.ID,
+		"session": session.ID,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	http.SetCookie(c.Writer, &http.Cookie{
+		Name:  "semaphore",
+		Value: encoded,
+		Path:  "/",
+	})
+
 	c.AbortWithStatus(204)
 }
 
 func logout(c *gin.Context) {
-	session := c.MustGet("session").(models.Session)
+	c.SetCookie("semaphore", "", -1, "/", "", false, true)
-	if err := database.Redis.Del(session.ID).Err(); err != nil {
-		panic(err)
-	}
-
 	c.AbortWithStatus(204)
 }

routes/router.go

@@ -21,14 +21,12 @@ func Route(r *gin.Engine) {
 	// set up the namespace
 	api := r.Group("/api")
 
-	api.Use(authentication)
-
 	func(api *gin.RouterGroup) {
 		api.POST("/login", login)
 		api.POST("/logout", logout)
 	}(api.Group("/auth"))
 
-	api.Use(MustAuthenticate)
+	api.Use(authentication)
 
 	api.GET("/ws", sockets.Handler)
 

routes/user.go

@@ -45,13 +45,6 @@ func createAPIToken(c *gin.Context) {
 		panic(err)
 	}
 
-	temp_session := models.Session{
-		UserID: &user.ID,
-	}
-	if err := database.Redis.Set("token-session:"+token.ID, temp_session.Encode(), 0).Err(); err != nil {
-		panic(err)
-	}
-
 	c.JSON(201, token)
 }
 
@@ -69,11 +62,9 @@ func expireAPIToken(c *gin.Context) {
 		panic(err)
 	}
 
-	if affected > 0 {
+	if affected == 0 {
-		// remove from redis
+		c.AbortWithStatus(400)
-		if err := database.Redis.Del("token-session:" + tokenID).Err(); err != nil {
+		return
-			panic(err)
-		}
 	}
 
 	c.AbortWithStatus(204)

util/config.go

@@ -1,20 +1,20 @@
 package util
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"path"
 
-	"golang.org/x/crypto/bcrypt"
-
 	"github.com/bugsnag/bugsnag-go"
 	"github.com/gin-gonic/gin"
-	"github.com/mattbaird/gochimp"
+	"github.com/gorilla/securecookie"
+	"golang.org/x/crypto/bcrypt"
 )
 
-var mandrillAPI *gochimp.MandrillAPI
+var Cookie *securecookie.SecureCookie
 var Migration bool
 var InteractiveSetup bool
 var Upgrade bool
@@ -26,25 +26,25 @@ type mySQLConfig struct {
 	DbName   string `json:"name"`
 }
 
-type mandrillConfig struct {
-	Username string `json:"username"`
-	Password string `json:"password"`
-}
-
 type configType struct {
 	MySQL mySQLConfig `json:"mysql"`
-	// Format as is with net.Dial
-	SessionDb string         `json:"session_db"`
-	Mandrill  mandrillConfig `json:"mandrill"`
 	// Format `:port_num` eg, :3000
 	Port       string `json:"port"`
 	BugsnagKey string `json:"bugsnag_key"`
 
 	// semaphore stores projects here
 	TmpPath string `json:"tmp_path"`
+
+	// cookie hashing & encryption
+	CookieHash       string `json:"cookie_hash"`
+	CookieEncryption string `json:"cookie_encryption"`
 }
 
-var Config configType
+var Config *configType
+
+func NewConfig() *configType {
+	return &configType{}
+}
 
 func init() {
 	flag.BoolVar(&InteractiveSetup, "setup", false, "perform interactive setup")
@@ -61,16 +61,18 @@ func init() {
 	flag.Parse()
 
 	if printConfig {
-		b, _ := json.MarshalIndent(&configType{
+		cfg := &configType{
 			MySQL: mySQLConfig{
 				Hostname: "127.0.0.1:3306",
 				Username: "root",
 				DbName:   "semaphore",
 			},
-			SessionDb: "127.0.0.1:6379",
 			Port:    ":3000",
 			TmpPath: "/tmp/semaphore",
-		}, "", "\t")
+		}
+		cfg.GenerateCookieSecrets()
+
+		b, _ := json.MarshalIndent(cfg, "", "\t")
 		fmt.Println(string(b))
 
 		os.Exit(0)
@@ -114,15 +116,20 @@ func init() {
 		Config.Port = ":3000"
 	}
 
-	if len(Config.Mandrill.Password) > 0 {
-		api, _ := gochimp.NewMandrill(Config.Mandrill.Password)
-		mandrillAPI = api
-	}
-
 	if len(Config.TmpPath) == 0 {
 		Config.TmpPath = "/tmp/semaphore"
 	}
 
+	var encryption []byte
+	encryption = nil
+
+	hash, _ := base64.StdEncoding.DecodeString(Config.CookieHash)
+	if len(Config.CookieEncryption) > 0 {
+		encryption, _ = base64.StdEncoding.DecodeString(Config.CookieEncryption)
+	}
+
+	Cookie = securecookie.New(hash, encryption)
+
 	stage := ""
 	if gin.Mode() == "release" {
 		stage = "production"
@@ -138,33 +145,15 @@ func init() {
 	})
 }
 
-// encapsulate mandrill providing some defaults
+func (conf *configType) GenerateCookieSecrets() {
+	hash := securecookie.GenerateRandomKey(32)
+	encryption := securecookie.GenerateRandomKey(32)
 
-func MandrillMessage(important bool) gochimp.Message {
+	conf.CookieHash = base64.StdEncoding.EncodeToString(hash)
-	return gochimp.Message{
+	conf.CookieEncryption = base64.StdEncoding.EncodeToString(encryption)
-		AutoText:  true,
-		InlineCss: true,
-		Important: important,
-		FromName:  "Semaphore Daemon",
-		FromEmail: "noreply@semaphore.local",
-	}
 }
 
-func MandrillRecipient(name string, email string) gochimp.Recipient {
+func (conf *configType) Scan() {
-	return gochimp.Recipient{
-		Email: email,
-		Name:  name,
-		Type:  "to",
-	}
-}
-
-func MandrillSend(message gochimp.Message) ([]gochimp.SendResponse, error) {
-	return mandrillAPI.MessageSend(message, false)
-}
-
-func ScanSetup() configType {
-	var conf configType
-
 	fmt.Print(" > DB Hostname (default 127.0.0.1:3306): ")
 	fmt.Scanln(&conf.MySQL.Hostname)
 	if len(conf.MySQL.Hostname) == 0 {
@@ -186,12 +175,6 @@ func ScanSetup() configType {
 		conf.MySQL.DbName = "semaphore"
 	}
 
-	fmt.Print(" > Redis Connection (default 127.0.0.1:6379): ")
-	fmt.Scanln(&conf.SessionDb)
-	if len(conf.SessionDb) == 0 {
-		conf.SessionDb = "127.0.0.1:6379"
-	}
-
 	fmt.Print(" > Playbook path: ")
 	fmt.Scanln(&conf.TmpPath)
 
@@ -199,6 +182,4 @@ func ScanSetup() configType {
 		conf.TmpPath = "/tmp/semaphore"
 	}
 	conf.TmpPath = path.Clean(conf.TmpPath)
-
-	return conf
 }