From cccc00d11369d29907fddcfdc8244de311200f7b Mon Sep 17 00:00:00 2001 From: tom whiston Date: Wed, 11 Apr 2018 18:05:38 +0000 Subject: [PATCH] use dredd for api testing add ci context docker deployment update api docs add some small fixes --- .circleci/config.yml | 55 ++- .dredd/dredd.local.yml | 34 ++ .dredd/dredd.yml | 34 ++ .dredd/hooks/capabilities.go | 159 ++++++++ .dredd/hooks/helpers.go | 192 ++++++++++ .dredd/hooks/main.go | 120 ++++++ CONTRIBUTING.md | 33 ++ Gopkg.lock | 142 ++++++- Taskfile.yml | 27 +- api-docs.yml | 468 ++++++++++++++++------- api/api_test.go | 33 ++ api/projects/projects.go | 5 + api/projects/users.go | 1 + api/router.go | 15 +- api/user.go | 2 +- api/users.go | 3 +- db/AccessKey.go | 2 +- db/mysql.go | 18 +- deployment/docker/Readme.md | 7 + deployment/docker/ci/Dockerfile | 36 ++ deployment/docker/ci/bin/install | 15 + deployment/docker/ci/docker-compose.yml | 45 +++ deployment/docker/ci/dredd.Dockerfile | 24 ++ deployment/docker/ci/dredd/entrypoint | 25 ++ deployment/docker/dev/Dockerfile | 2 +- deployment/docker/dev/bin/install | 2 + deployment/docker/dev/docker-compose.yml | 2 + deployment/docker/prod/bin/install | 4 + 28 files changed, 1338 insertions(+), 167 deletions(-) create mode 100644 .dredd/dredd.local.yml create mode 100644 .dredd/dredd.yml create mode 100644 .dredd/hooks/capabilities.go create mode 100644 .dredd/hooks/helpers.go create mode 100644 .dredd/hooks/main.go create mode 100644 api/api_test.go create mode 100644 deployment/docker/ci/Dockerfile create mode 100755 deployment/docker/ci/bin/install create mode 100644 deployment/docker/ci/docker-compose.yml create mode 100644 deployment/docker/ci/dredd.Dockerfile create mode 100755 deployment/docker/ci/dredd/entrypoint diff --git a/.circleci/config.yml b/.circleci/config.yml index 242ea928..8eeb08d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ aliases: image: circleci/golang:1.10 - &working-dir - /go/src/github.com/ansible-semaphore/semaphore + /go/src/github.com/ansible-semaphore/semaphore - &store-bin-artifacts store_artifacts: @@ -21,7 +21,7 @@ aliases: curl -L https://github.com/go-task/task/releases/download/v2.0.1/task_linux_amd64.tar.gz | tar xvz cd - - - &persist-bin + - &persist-from-build persist_to_workspace: root: . paths: @@ -79,12 +79,12 @@ aliases: - v1-go-deps- jobs: + build:local: docker: - *golang-image working_directory: *working-dir steps: - - run: export - *install-node - *install-task-binary - checkout @@ -97,7 +97,7 @@ jobs: - *test-compile-changes - run: task build:local - *store-bin-artifacts - - *persist-bin + - *persist-from-build build: docker: @@ -117,6 +117,22 @@ jobs: - run: task build - *store-bin-artifacts + test:integration:hooks: + docker: + - *golang-image + working_directory: *working-dir + steps: + - checkout + - *install-node + - *install-task-binary + - run: task deps:integration + - run: task deps:tools + - run: task deps:be + - run: task compile:be + - run: task compile:api:hooks + - store_artifacts: + path: /go/src/github.com/ansible-semaphore/semaphore/.dredd/compiled_hooks + # Run goverage and post results test:golang: docker: @@ -140,11 +156,23 @@ jobs: path: /go/src/github.com/ansible-semaphore/semaphore/coverage.out test:integration: + machine: true + steps: + - checkout + - run: | + cd /home/circleci/bin + sudo curl -L https://github.com/go-task/task/releases/download/v2.0.1/task_linux_amd64.tar.gz | tar xvz + cd - + - run: context=ci task dc:up + + test:db:migration: docker: - *golang-image - image: circleci/mysql working_directory: *working-dir steps: + - *install-task-binary + - *install-node - attach_workspace: at: *working-dir # This looks like utter filth in circleci v2 but we have no choice apart from this escaping madness @@ -155,7 +183,6 @@ jobs: name: Wait for db command: dockerize -wait tcp://127.0.0.1:3306 -timeout 1m - run: bin/semaphore --migrate -config config.json - # TODO - Here we could do some api/functional testing test:docker: docker: @@ -221,8 +248,10 @@ workflows: jobs: - test:docker - test:golang + - test:integration:hooks + - test:integration - build:local - - test:integration: + - test:db:migration: requires: - build:local @@ -230,6 +259,7 @@ workflows: - build: requires: - test:golang + - test:db:migration - test:integration filters: branches: @@ -244,12 +274,11 @@ workflows: branches: only: develop -# Production releases only run on tags -# use _ in tags and not - due to rpmbuild not allowing hyphens in version numbers -# https://github.com/semver/semver/issues/145 - release: - jobs: +# Production deploys only happen if everything passes +# and we have a tag starting with v - release: + requires: + - build filters: branches: ignore: /.*/ @@ -257,9 +286,11 @@ workflows: only: /^v.*/ - deploy:prod: + requires: + - build + - test:docker filters: branches: ignore: /.*/ tags: only: /^v.*/ - \ No newline at end of file diff --git a/.dredd/dredd.local.yml b/.dredd/dredd.local.yml new file mode 100644 index 00000000..90d0bea7 --- /dev/null +++ b/.dredd/dredd.local.yml @@ -0,0 +1,34 @@ +dry-run: null +hookfiles: ./.dredd/compiled_hooks +language: go +sandbox: false +#server: context=dev task dc:up +server-wait: 240 +init: false +custom: {} +names: false +only: [] +reporter: [] +output: [] +header: "Authorization: bearer h4a_i4qslpnxyyref71rk5nqbwxccrs7enwvggx0vfs=" +sorted: false +user: null +inline-errors: false +details: false +method: [] +color: true +level: info +timestamp: false +silent: false +path: [] +hooks-worker-timeout: 5000 +hooks-worker-connect-timeout: 1500 +hooks-worker-connect-retry: 500 +hooks-worker-after-connect-wait: 100 +hooks-worker-term-timeout: 5000 +hooks-worker-term-retry: 500 +hooks-worker-handler-host: 0.0.0.0 +hooks-worker-handler-port: 61321 +config: ./.dredd/dredd.yml +blueprint: api-docs.yml +endpoint: 'http://0.0.0.0:3000' diff --git a/.dredd/dredd.yml b/.dredd/dredd.yml new file mode 100644 index 00000000..0681f4c7 --- /dev/null +++ b/.dredd/dredd.yml @@ -0,0 +1,34 @@ +dry-run: null +hookfiles: ./.dredd/compiled_hooks +language: go +sandbox: false +#server: context=dev task dc:up +server-wait: 240 +init: false +custom: {} +names: false +only: [] +reporter: [] +output: [] +header: "Authorization: bearer h4a_i4qslpnxyyref71rk5nqbwxccrs7enwvggx0vfs=" +sorted: false +user: null +inline-errors: false +details: false +method: [] +color: true +level: info +timestamp: false +silent: false +path: [] +hooks-worker-timeout: 5000 +hooks-worker-connect-timeout: 1500 +hooks-worker-connect-retry: 500 +hooks-worker-after-connect-wait: 100 +hooks-worker-term-timeout: 5000 +hooks-worker-term-retry: 500 +hooks-worker-handler-host: 0.0.0.0 +hooks-worker-handler-port: 61321 +config: ./.dredd/dredd.yml +blueprint: api-docs.yml +endpoint: 'http://semaphore_ci:3000' diff --git a/.dredd/hooks/capabilities.go b/.dredd/hooks/capabilities.go new file mode 100644 index 00000000..f431cb39 --- /dev/null +++ b/.dredd/hooks/capabilities.go @@ -0,0 +1,159 @@ +package main + +import ( + "encoding/json" + "github.com/ansible-semaphore/semaphore/db" + trans "github.com/snikch/goodman/transaction" + "strconv" + "strings" +) + +// STATE +// Runtime created objects we needs to reference in test setups +var testRunnerUser *db.User +var userPathTestUser *db.User +var userProject *db.Project +var userKey *db.AccessKey +var task *db.Task + +// Runtime created simple ID values for some items we need to reference in other objects +var repoID int64 +var inventoryID int64 +var environmentID int64 +var templateID int64 + +var capabilities = map[string][]string{ + "user": {}, + "project": {"user"}, + "access_key": {"project"}, + "repository": {"access_key"}, + "inventory": {"repository"}, + "environment": {"repository"}, + "template": {"repository", "inventory", "environment"}, + "task": {"template"}, +} + +func capabilityWrapper(cap string) func(t *trans.Transaction) { + return func(t *trans.Transaction) { + addCapabilities([]string{cap}) + } +} + +func addCapabilities(caps []string) { + dbConnect() + defer db.Mysql.Db.Close() + resolved := make([]string, 0) + uid := getUUID() + resolveCapability(caps, resolved, uid) +} + +func resolveCapability(caps []string, resolved []string, uid string) { + for _, v := range caps { + + //if cap has deps resolve them + if val, ok := capabilities[v]; ok { + resolveCapability(val, resolved, uid) + } + + //skip if already resolved + if _, exists := stringInSlice(v, resolved); exists { + continue + } + + //Add dep specific stuff + switch v { + case "user": + userPathTestUser = addUser() + case "project": + userProject = addProject() + //allow the admin user (test executor) to manipulate the project + addUserProjectRelation(userProject.ID, testRunnerUser.ID) + addUserProjectRelation(userProject.ID, userPathTestUser.ID) + case "access_key": + userKey = addAccessKey(&userProject.ID) + case "repository": + pRepo, err := db.Mysql.Exec("insert into project__repository set project_id=?, git_url=?, ssh_key_id=?, name=?", userProject.ID, "git@github.com/ansible,semaphore/semaphore", userKey.ID, "ITR-"+uid) + printError(err) + repoID, _ = pRepo.LastInsertId() + case "inventory": + res, err := db.Mysql.Exec("insert into project__inventory set project_id=?, name=?, type=?, key_id=?, ssh_key_id=?, inventory=?", userProject.ID, "ITI-"+uid, "static", userKey.ID, userKey.ID, "Test Inventory") + printError(err) + inventoryID, _ = res.LastInsertId() + case "environment": + res, err := db.Mysql.Exec("insert into project__environment set project_id=?, name=?, json=?, password=?", userProject.ID, "ITI-"+uid, "{}", "test-pass") + printError(err) + environmentID, _ = res.LastInsertId() + case "template": + res, err := db.Mysql.Exec("insert into project__template set ssh_key_id=?, project_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=?", userKey.ID, userProject.ID, inventoryID, repoID, environmentID, "Test-"+uid, "test-playbook.yml", "", false) + printError(err) + templateID, _ = res.LastInsertId() + case "task": + task = addTask() + } + resolved = append(resolved, v) + } +} + +// HOOKS +var skipTest = func(t *trans.Transaction) { + t.Skip = true +} + +// Contains all the substitutions for paths under test +// The parameter example value in the api-doc should respond to the index+1 of the function in this slice +// ie the project id, with example value 1, will be replaced by the return value of pathSubPatterns[0] +var pathSubPatterns = []func() string{ + func() string { return strconv.Itoa(userProject.ID) }, + func() string { return strconv.Itoa(userPathTestUser.ID) }, + func() string { return strconv.Itoa(userKey.ID) }, + func() string { return strconv.Itoa(int(repoID)) }, + func() string { return strconv.Itoa(int(inventoryID)) }, + func() string { return strconv.Itoa(int(environmentID)) }, + func() string { return strconv.Itoa(int(templateID)) }, + func() string { return strconv.Itoa(task.ID) }, +} + +// alterRequestPath with the above slice of functions +func alterRequestPath(t *trans.Transaction) { + pathArgs := strings.Split(t.FullPath, "/") + exploded := make([]string, len(pathArgs)) + copy(exploded, pathArgs) + for k, v := range pathSubPatterns { + pos, exists := stringInSlice(strconv.Itoa(k+1), exploded) + if exists { + pathArgs[pos] = v() + } + } + t.FullPath = strings.Join(pathArgs, "/") + t.Request.URI = t.FullPath +} + +func alterRequestBody(t *trans.Transaction) { + var request map[string]interface{} + json.Unmarshal([]byte(t.Request.Body), &request) + + if userProject != nil { + bodyFieldProcessor("project_id", userProject.ID, &request) + } + bodyFieldProcessor("json", "{}", &request) + if userKey != nil { + bodyFieldProcessor("ssh_key_id", userKey.ID, &request) + bodyFieldProcessor("key_id", userKey.ID, &request) + } + bodyFieldProcessor("environment_id", environmentID, &request) + bodyFieldProcessor("inventory_id", inventoryID, &request) + bodyFieldProcessor("repository_id", repoID, &request) + bodyFieldProcessor("template_id", templateID, &request) + if task != nil { + bodyFieldProcessor("task_id", task.ID, &request) + } + + out, _ := json.Marshal(request) + t.Request.Body = string(out) +} + +func bodyFieldProcessor(id string, sub interface{}, request *map[string]interface{}) { + if _, ok := (*request)[id]; ok { + (*request)[id] = sub + } +} diff --git a/.dredd/hooks/helpers.go b/.dredd/hooks/helpers.go new file mode 100644 index 00000000..81ed9dfc --- /dev/null +++ b/.dredd/hooks/helpers.go @@ -0,0 +1,192 @@ +package main + +import ( + "encoding/json" + "fmt" + "github.com/ansible-semaphore/semaphore/db" + "github.com/ansible-semaphore/semaphore/util" + "github.com/snikch/goodman/transaction" + "math/rand" + "os" + "strconv" + "time" +) + +// Test Runner User +func addTestRunnerUser() { + uid := getUUID() + testRunnerUser = &db.User{ + Username: "ITU-" + uid, + Name: "ITU-" + uid, + Email: uid + "@semaphore.test", + Created: db.GetParsedTime(time.Now()), + Admin: true, + } + + dbConnect() + defer db.Mysql.Db.Close() + if err := db.Mysql.Insert(testRunnerUser); err != nil { + panic(err) + } + addToken(adminToken, testRunnerUser.ID) +} + +func removeTestRunnerUser(transactions []*transaction.Transaction) { + dbConnect() + defer db.Mysql.Db.Close() + deleteToken(adminToken, testRunnerUser.ID) + deleteObject(testRunnerUser) +} + +// Parameter Substitution +func setupObjectsAndPaths(t *transaction.Transaction) { + alterRequestBody(t) + alterRequestPath(t) +} + +// Object Lifecycle +func addUserProjectRelation(pid int, user int) { + _, err := db.Mysql.Exec("insert into project__user set project_id=?, user_id=?, `admin`=1", pid, user) + if err != nil { + fmt.Println(err) + } +} +func deleteUserProjectRelation(pid int, user int) { + _, err := db.Mysql.Exec("delete from project__user where project_id=? and user_id=?", strconv.Itoa(pid), strconv.Itoa(user)) + if err != nil { + fmt.Println(err) + } +} + +func addAccessKey(pid *int) *db.AccessKey { + uid := getUUID() + secret := "5up3r53cr3t" + key := db.AccessKey{ + Name: "ITK-" + uid, + Type: "ssh", + Secret: &secret, + ProjectID: pid, + } + if err := db.Mysql.Insert(&key); err != nil { + fmt.Println(err) + } + return &key +} + +func addProject() *db.Project { + uid := getUUID() + project := db.Project{ + Name: "ITP-" + uid, + Created: time.Now(), + } + if err := db.Mysql.Insert(&project); err != nil { + fmt.Println(err) + } + return &project +} + +func addUser() *db.User { + uid := getUUID() + user := db.User{ + Created: time.Now(), + Username: "ITU-" + uid, + Email: "test@semaphore." + uid, + } + if err := db.Mysql.Insert(&user); err != nil { + fmt.Println(err) + } + return &user +} + +func addTask() *db.Task { + t := db.Task{ + TemplateID: int(templateID), + Status: "testing", + UserID: &userPathTestUser.ID, + Created: db.GetParsedTime(time.Now()), + } + if err := db.Mysql.Insert(&t); err != nil { + fmt.Println(err) + } + return &t +} + +func deleteObject(i interface{}) { + _, err := db.Mysql.Delete(i) + if err != nil { + fmt.Println(err) + } +} + +// Token Handling +func addToken(tok string, user int) { + token := db.APIToken{ + ID: tok, + Created: time.Now(), + UserID: user, + Expired: false, + } + if err := db.Mysql.Insert(&token); err != nil { + fmt.Println(err) + } +} + +func deleteToken(tok string, user int) { + token := db.APIToken{ + ID: tok, + UserID: user, + } + deleteObject(&token) +} + +// HELPERS +var r *rand.Rand +var randSetup = false + +func getUUID() string { + if !randSetup { + r = rand.New(rand.NewSource(time.Now().UnixNano())) + randSetup = true + } + return randomString(8) +} +func randomString(strlen int) string { + const chars = "abcdefghijklmnopqrstuvwxyz0123456789" + result := "" + for i := 0; i < strlen; i++ { + index := r.Intn(len(chars)) + result += chars[index : index+1] + } + return result +} + +func loadConfig() { + cwd, _ := os.Getwd() + file, _ := os.Open(cwd + "/.dredd/config.json") + if err := json.NewDecoder(file).Decode(&util.Config); err != nil { + fmt.Println("Could not decode configuration!") + panic(err) + } +} + +func dbConnect() { + if err := db.Connect(); err != nil { + panic(err) + } + db.SetupDBLink() +} + +func stringInSlice(a string, list []string) (int, bool) { + for k, b := range list { + if b == a { + return k, true + } + } + return 0, false +} + +func printError(err error) { + if err != nil { + fmt.Println(err) + } +} diff --git a/.dredd/hooks/main.go b/.dredd/hooks/main.go new file mode 100644 index 00000000..fcfc9106 --- /dev/null +++ b/.dredd/hooks/main.go @@ -0,0 +1,120 @@ +package main + +import ( + "github.com/ansible-semaphore/semaphore/db" + "github.com/snikch/goodman/hooks" + trans "github.com/snikch/goodman/transaction" + "strconv" + "strings" +) + +const ( + adminToken = "h4a_i4qslpnxyyref71rk5nqbwxccrs7enwvggx0vfs=" + expiredToken = "kwofd61g93-yuqvex8efmhjkgnbxlo8mp1tin6spyhu=" +) + +var skipTests = []string{ + // TODO - dredd seems not to like the text response from this endpoint + "/api/ping > PING test > 200 > text/plain; charset=utf-8", + "/api/ws > Websocket handler > 200 > application/json", + "authentication > /api/auth/login > Performs Login > 204 > application/json", + "authentication > /api/auth/logout > Destroys current session > 204 > application/json", + "/api/upgrade > Upgrade the server > 200 > application/json", + // TODO - Skipping this while we work out how to get a 204 response from the api for testing + "/api/upgrade > Check if new updates available and fetch /info > 204 > application/json", +} + +// Dredd expects that you have already set up the database and run all migrations before it begins. +// It will NOT initialize the database, only insert its test data. +// It does this in a way which ignores errors, which is fine on the ci, but might be an issue locally +// so look at the logs carefully if these tests fail and if in doubt re-init the db +// These hooks do NOT clean up after themselves and they produce a lot of database writes, +// so don't run this in production +func main() { + + h := hooks.NewHooks() + server := hooks.NewServer(hooks.NewHooksRunner(h)) + + //Get database connection info and create an admin who's token is used to execute the tests + h.BeforeAll(func(t []*trans.Transaction) { + loadConfig() + addTestRunnerUser() + }) + + for _, v := range skipTests { + h.Before(v, skipTest) + } + + h.BeforeEach(func(t *trans.Transaction) { + if strings.HasPrefix(t.Name, "user") { + addCapabilities([]string{"user"}) + } else if strings.HasPrefix(t.Name, "project") || strings.HasPrefix(t.Name, "projects") { + addCapabilities([]string{"project"}) + } + }) + + h.Before("user > /api/user/tokens/{api_token_id} > Expires API token > 204 > application/json", func(transaction *trans.Transaction) { + dbConnect() + defer db.Mysql.Db.Close() + addToken(expiredToken, testRunnerUser.ID) + }) + h.After("user > /api/user/tokens/{api_token_id} > Expires API token > 204 > application/json", func(transaction *trans.Transaction) { + dbConnect() + defer db.Mysql.Db.Close() + //tokens are expired and not deleted so we need to clean up + deleteToken(expiredToken, testRunnerUser.ID) + }) + + // This one seems to need some manual value setting in the body + h.Before("user > /api/users/{user_id}/password > Updates user password > 204 > application/json", func(transaction *trans.Transaction) { + transaction.Request.Body = "{\"password\":\"staub\"}" + }) + + // delete the auto generated association and insert the user id into the query + h.Before("project > /api/project/{project_id}/users > Link user to project > 204 > application/json", func(transaction *trans.Transaction) { + dbConnect() + defer db.Mysql.Db.Close() + deleteUserProjectRelation(userProject.ID, userPathTestUser.ID) + transaction.Request.Body = "{ \"user_id\": " + strconv.Itoa(userPathTestUser.ID) + ",\"admin\": true}" + }) + + h.Before("project > /api/project/{project_id}/keys/{key_id} > Updates access key > 204 > application/json", capabilityWrapper("access_key")) + h.Before("project > /api/project/{project_id}/keys/{key_id} > Removes access key > 204 > application/json", capabilityWrapper("access_key")) + + h.Before("project > /api/project/{project_id}/repositories > Add repository > 204 > application/json", capabilityWrapper("access_key")) + h.Before("project > /api/project/{project_id}/repositories/{repository_id} > Removes repository > 204 > application/json", capabilityWrapper("repository")) + + h.Before("project > /api/project/{project_id}/inventory > create inventory > 201 > application/json", capabilityWrapper("inventory")) + h.Before("project > /api/project/{project_id}/inventory/{inventory_id} > Updates inventory > 204 > application/json", capabilityWrapper("inventory")) + h.Before("project > /api/project/{project_id}/inventory/{inventory_id} > Removes inventory > 204 > application/json", capabilityWrapper("inventory")) + + h.Before("project > /api/project/{project_id}/environment/{environment_id} > Update environment > 204 > application/json", capabilityWrapper("environment")) + h.Before("project > /api/project/{project_id}/environment/{environment_id} > Removes environment > 204 > application/json", capabilityWrapper("environment")) + + h.Before("project > /api/project/{project_id}/templates > create template > 201 > application/json", func(t *trans.Transaction) { + addCapabilities([]string{"repository", "inventory", "environment"}) + }) + + h.Before("project > /api/project/{project_id}/templates/{template_id} > Updates template > 204 > application/json", capabilityWrapper("template")) + h.Before("project > /api/project/{project_id}/templates/{template_id} > Removes template > 204 > application/json", capabilityWrapper("template")) + + h.Before("project > /api/project/{project_id}/tasks > Starts a job > 201 > application/json", capabilityWrapper("template")) + h.Before("project > /api/project/{project_id}/tasks/last > Get last 200 Tasks related to current project > 200 > application/json", capabilityWrapper("template")) + + h.Before("project > /api/project/{project_id}/tasks/{task_id} > Get a single task > 200 > application/json", capabilityWrapper("task")) + h.Before("project > /api/project/{project_id}/tasks/{task_id} > Deletes task (including output) > 204 > application/json", capabilityWrapper("task")) + h.Before("project > /api/project/{project_id}/tasks/{task_id}/output > Get task output > 200 > application/json", capabilityWrapper("task")) + + //Add these last as they normalize the requests and path values after hook processing + h.BeforeAll(func(transactions []*trans.Transaction) { + for _, t := range transactions { + h.Before(t.Name, setupObjectsAndPaths) + } + }) + + // Delete the test runner user so adding him next time does not result in errors + h.AfterAll(removeTestRunnerUser) + + server.Serve() + defer server.Listener.Close() +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 03b5a8fb..900578b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,7 @@ When creating a pull-request you should: - __gofmt and vet the code:__ Use `gofmt`, `golint`, `govet` and `goimports` to clean up your code. - __vendor dependencies with dep:__ Use `dep ensure --update` if you have added to or updated dependencies, so that they get added to the dependency manifest. - __Update api documentation:__ If your pull-request adding/modifying an API request, make sure you update the swagger documentation (`api-docs.yml`) +- __Run Api Tests:__ If your pull request modifies the API make sure you run the integration tests using dredd. # Installation in a development environment @@ -70,3 +71,35 @@ Now it's ready to start.. Run `task watch` Note: for Windows, you may need [Cygwin](https://www.cygwin.com/) to run certain commands because the [reflex](github.com/cespare/reflex) package probably doesn't work on Windows. You may encounter issues when running `task watch`, but running `task build` etc... will still be OK. + +## Integration Tests + +Dredd is used for API integration tests, if you alter the API in any way you must make sure that the information in the api docs +matches the responses. + +As Dredd and the application database config may differ it expects it's own config.json in the .dredd folder. +The most basic configuration for this using a local docker container to run the database would be +```json +{ + "mysql": { + "host": "0.0.0.0:3306", + "user": "semaphore", + "pass": "semaphore", + "name": "semaphore" + } +} + +``` + +It is strongly advised to run these tests inside docker containers, as dredd will write a lot of test information and will __NOT__ clear it up. +This means that you should never run these tests against your productive database! +The best practice to run these tests is to use docker and the task commands. + +```bash +task dc:build #First run only to build the images +context=dev task dc:up +task test:api +# alternatively if you want to run dredd in a container use the following command. +# You will need to use the host network so that it can reach the docker container on a 0.0.0.0 address +# docker run -it --rm -v $(pwd):/home/developer/src --network host tomwhiston/dredd --config .dredd/dredd.yml +``` \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock index 701bd5ed..bca322d1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,18 +1,96 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + name = "github.com/PuerkitoBio/purell" + packages = ["."] + revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/PuerkitoBio/urlesc" + packages = ["."] + revision = "de5bf2ad457846296e2031421a34e2568e304e35" + [[projects]] name = "github.com/Sirupsen/logrus" packages = ["."] revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba" version = "v1.0.4" +[[projects]] + name = "github.com/asaskevich/govalidator" + packages = ["."] + revision = "ccb8e960c48f04d6935e72476ae4a51028f9e22f" + version = "v9" + [[projects]] branch = "master" name = "github.com/castawaylabs/mulekick" packages = ["."] revision = "7029fb389811e0f873c56cfbbda64d66af48b095" +[[projects]] + branch = "master" + name = "github.com/go-openapi/analysis" + packages = ["."] + revision = "f59a71f0ece6f9dfb438be7f45148f006cbad88e" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/errors" + packages = ["."] + revision = "7bcb96a367bac6b76e6e42fa84155bb5581dcff8" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonpointer" + packages = ["."] + revision = "3a0015ad55fa9873f41605d3e8f28cd279c32ab2" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonreference" + packages = ["."] + revision = "3fb327e6747da3043567ee86abd02bb6376b6be2" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/loads" + packages = ["."] + revision = "2a2b323bab96e6b1fdee110e57d959322446e9c9" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/runtime" + packages = ["."] + revision = "62281b694b396a17fe3e4313ee8b0ca2c3cca719" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/spec" + packages = ["."] + revision = "370d9e047557906322be8396e77cb0376be6cb96" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/strfmt" + packages = ["."] + revision = "481808443b00a14745fada967cb5eeff0f9b1df2" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/swag" + packages = ["."] + revision = "811b1089cde9dad18d4d0c2d09fbdbf28dbd27a5" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/validate" + packages = ["."] + revision = "180bba53b98899f743a112e568bed9e2ef31aa20" + [[projects]] name = "github.com/go-sql-driver/mysql" packages = ["."] @@ -79,12 +157,28 @@ packages = ["."] revision = "62de8c46ede02a7675c4c79c84883eb164cb71e3" +[[projects]] + branch = "master" + name = "github.com/mailru/easyjson" + packages = [ + "buffer", + "jlexer", + "jwriter" + ] + revision = "8b799c424f57fa123fc63a99d6383bc6e4c02578" + [[projects]] name = "github.com/masterminds/squirrel" packages = ["."] revision = "a6b93000bd219143c56c16e6cb1c4b91da3f224b" version = "v1.0" +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "00c29f56e2386353d58c599509e8dc3801b0d716" + [[projects]] name = "github.com/pkg/errors" packages = ["."] @@ -107,6 +201,15 @@ ] revision = "c7dcf104e3a7a1417abc0230cb0d5240d764159d" +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "idna" + ] + revision = "61147c48b25b599e5b561d2e9c4f3e1ef489ca41" + [[projects]] branch = "master" name = "golang.org/x/sys" @@ -116,6 +219,28 @@ ] revision = "7dca6fe1f43775aa6d1334576870ff63f978f539" +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable", + "width" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + [[projects]] name = "gopkg.in/asn1-ber.v1" packages = ["."] @@ -134,9 +259,24 @@ revision = "bb7a9ca6e4fbc2129e3db588a34bc970ffe811a9" version = "v2.5.1" +[[projects]] + branch = "v2" + name = "gopkg.in/mgo.v2" + packages = [ + "bson", + "internal/json" + ] + revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "fd98c9632d4a76491d66eb42e4dd5e6440b36405c061136afe7eb92196055d0f" + inputs-digest = "c07a879b1ce764530cc0e0a4f80e207f8a6f7a8f1bd3715c35fa6cd366822180" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Taskfile.yml b/Taskfile.yml index 7b080ce4..cf0e3606 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -19,7 +19,7 @@ tasks: - task: build:local deps: - desc: Install all dependencies + desc: Install all dependencies (except dredd requirements) cmds: - task: deps:tools - task: deps:be @@ -36,16 +36,23 @@ tasks: cmds: - npm install + deps:integration: + desc: Installs requirements for integration testing with dredd + dir: web + cmds: + - npm install dredd@5.1.5 + deps:tools: desc: Installs tools needed dir: web vars: - GORELEASER_VERSION: "0.66.1" + GORELEASER_VERSION: "0.67.0" cmds: - go get -u github.com/golang/dep/cmd/dep - go get github.com/cespare/reflex || true - go get -u github.com/gobuffalo/packr/... - go get -u github.com/haya14busa/goverage + - go get github.com/snikch/goodman/cmd/goodman - '{{ if ne OS "windows" }} curl -L https://github.com/goreleaser/goreleaser/releases/download/v{{ .GORELEASER_VERSION }}/goreleaser_$(uname -s)_$(uname -m).tar.gz | tar -xz -C ${GOPATH}/bin{{ else }} {{ end }}' - '{{ if ne OS "windows" }} chmod +x ${GOPATH}/bin/goreleaser{{ else }} {{ end }}' - '{{ if eq OS "windows" }} echo "NOTICE: You must download goreleaser manually to build this application https://github.com/goreleaser/goreleaser/releases "{{ else }}:{{ end }}' @@ -94,12 +101,17 @@ tasks: sh: git rev-parse --abbrev-ref HEAD DIRTY: # We must exclude the package-lock file as npm install can change it! - sh: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json' + sh: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json' ':(exclude)web/package.json' SHA: sh: git log --pretty=format:'%h' -n 1 TIMESTAMP: sh: date +%s + compile:api:hooks: + dir: ./.dredd/hooks + cmds: + - go build -o ../compiled_hooks + watch: desc: Watch fe and be file changes and rebuild dir: web/resources @@ -148,6 +160,10 @@ tasks: - gometalinter --exclude "\w*(-packr.go)" --vendor --disable goconst --deadline 240s ./... test: + cmds: + - task: test:be + + test:be: desc: Run go code tests cmds: - go vet ./... @@ -155,6 +171,11 @@ tasks: # if no tests exist but will still print failing test results - goverage -v -coverprofile=coverage.out ./... 2> /dev/null + test:api: + desc: test the api with dredd + cmds: + - ./web/node_modules/.bin/dredd --config .dredd/dredd.yml + ci:artifacts: cmds: - rsync -a bin/ $CIRCLE_ARTIFACTS/ diff --git a/api-docs.yml b/api-docs.yml index 884d3e3f..43d42793 100644 --- a/api-docs.yml +++ b/api-docs.yml @@ -6,6 +6,12 @@ info: host: localhost:3000 +consumes: + - application/json +produces: + - application/json + - text/plain; charset=utf-8 + tags: - name: authentication description: Authentication, Logout & API Tokens @@ -19,31 +25,47 @@ schemes: - https basePath: /api -produces: - - application/json definitions: + + Pong: + type: string + x-example: pong + Login: type: object - required: - - auth - - password properties: auth: type: string description: Username/Email address + x-example: user@semaphore.com password: type: string format: password description: Password - PONG: - type: string - format: PONG + + UserRequest: + type: object + properties: + name: + type: string + x-example: Integration Test User + username: + type: string + x-example: test-user + email: + type: string + x-example: test@ansiblesemaphore.test + alert: + type: boolean + admin: + type: boolean User: type: object properties: id: type: integer + minimum: 1 name: type: string username: @@ -52,11 +74,12 @@ definitions: type: string created: type: string - format: date-time + pattern: ^\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\d{2}:\d{2}:\d{2}Z$ alert: type: boolean admin: type: boolean + APIToken: type: object properties: @@ -64,23 +87,50 @@ definitions: type: string created: type: string - format: date-time + pattern: ^\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\d{2}:\d{2}:\d{2}Z$ expired: type: boolean user_id: type: integer + minimum: 1 + + ProjectRequest: + type: object + properties: + name: + type: string + alert: + type: boolean Project: type: object properties: id: type: integer + minimum: 1 name: type: string created: type: string - format: date-time + pattern: ^\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\d{2}:\d{2}:\d{2}Z$ alert: type: boolean + + AccessKeyRequest: + type: object + properties: + name: + type: string + type: + type: string + enum: [ssh, aws, gcloud, do] + project_id: + type: integer + minimum: 1 + x-example: 2 + key: + type: string + secret: + type: string AccessKey: type: object properties: @@ -90,25 +140,61 @@ definitions: type: string type: type: string + enum: [ssh, aws, gcloud, do] project_id: type: integer key: type: string secret: type: string + + EnvironmentRequest: + type: object + properties: + name: + type: string + project_id: + type: integer + minimum: 1 + password: + type: string + json: + type: string Environment: type: object properties: id: type: integer + minimum: 1 name: type: string project_id: type: integer + minimum: 1 password: type: string json: type: string + + InventoryRequest: + type: object + properties: + name: + type: string + project_id: + type: integer + minimum: 1 + inventory: + type: string + key_id: + type: integer + minimum: 1 + ssh_key_id: + type: integer + minimum: 1 + type: + type: string + enum: [static, file] Inventory: type: object properties: @@ -126,6 +212,19 @@ definitions: type: integer type: type: string + enum: [static, file] + + RepositoryRequest: + type: object + properties: + name: + type: string + project_id: + type: integer + git_url: + type: string + ssh_key_id: + type: integer Repository: type: object properties: @@ -139,11 +238,13 @@ definitions: type: string ssh_key_id: type: integer + Task: type: object properties: id: type: integer + example: 23 template_id: type: integer status: @@ -159,6 +260,7 @@ definitions: properties: task_id: type: integer + example: 23 task: type: string time: @@ -166,21 +268,25 @@ definitions: format: date-time output: type: string - Template: + + TemplateRequest: type: object properties: - id: - type: integer ssh_key_id: type: integer + minimum: 1 project_id: type: integer + minimum: 1 inventory_id: type: integer + minimum: 1 repository_id: type: integer + minimum: 1 environment_id: type: integer + minimum: 1 alias: type: string playbook: @@ -189,17 +295,51 @@ definitions: type: string override_args: type: boolean + Template: + type: object + properties: + id: + type: integer + minimum: 1 + ssh_key_id: + type: integer + minimum: 1 + project_id: + type: integer + minimum: 1 + inventory_id: + type: integer + minimum: 1 + repository_id: + type: integer + environment_id: + type: integer + minimum: 1 + alias: + type: string + playbook: + type: string + arguments: + type: string + override_args: + type: boolean + Event: type: object properties: project_id: type: integer object_id: - type: integer + type: + - integer + - 'null' object_type: - type: string + type: + - string + - 'null' description: type: string + InfoType: type: object properties: @@ -213,15 +353,19 @@ definitions: tag_name: type: string -# securityDefinitions: -# cookie: -# type: apiKey -# name: Cookie -# in: header -# bearer: -# type: apiKey -# name: Authorization -# in: header +securityDefinitions: + cookie: + type: apiKey + name: Cookie + in: header + bearer: + type: apiKey + name: Authorization + in: header + +security: + - bearer: [] + - cookie: [] parameters: project_id: @@ -230,58 +374,73 @@ parameters: in: path type: integer required: true + x-example: 1 user_id: name: user_id description: User ID in: path type: integer required: true + x-example: 2 key_id: name: key_id description: key ID in: path type: integer required: true + x-example: 3 repository_id: name: repository_id description: repository ID in: path type: integer required: true + x-example: 4 inventory_id: name: inventory_id description: inventory ID in: path type: integer required: true + x-example: 5 environment_id: name: environment_id description: environment ID in: path type: integer required: true + x-example: 6 template_id: name: template_id description: template ID in: path type: integer required: true + x-example: 7 task_id: name: task_id description: task ID in: path type: integer required: true + x-example: 8 paths: /ping: get: summary: PING test + produces: + - text/plain + security: [] # No security responses: 200: description: Successful "PONG" reply schema: - $ref: "#/definitions/PONG" + $ref: "#/definitions/Pong" + headers: + content-type: + type: string + x-example: text/plain; charset=utf-8 /ws: get: @@ -292,9 +451,9 @@ paths: responses: 200: description: OK - # security: - # - cookie: [] - # - bearer: [] + 403: + description: not authenticated + /info: get: summary: Fetches information about semaphore @@ -329,6 +488,7 @@ paths: summary: Performs Login description: | Upon success you will be logged in + security: [] # No security parameters: - name: Login Body in: body @@ -340,6 +500,7 @@ paths: description: You are logged in 400: description: something in body is missing / is invalid + /auth/logout: post: tags: @@ -349,7 +510,7 @@ paths: 204: description: Your session was successfully nuked - # User stuff + # User Tokens /user: get: tags: @@ -384,12 +545,14 @@ paths: description: API Token schema: $ref: "#/definitions/APIToken" + /user/tokens/{api_token_id}: parameters: - name: api_token_id in: path type: string required: true + x-example: "kwofd61g93-yuqvex8efmhjkgnbxlo8mp1tin6spyhu=" delete: tags: - authentication @@ -399,6 +562,98 @@ paths: 204: description: Expired API Token + # User Profiles + /users: + get: + tags: + - user + summary: Fetches all users + responses: + 200: + description: Users + schema: + type: array + items: + $ref: "#/definitions/User" + post: + tags: + - user + summary: Creates a user + consumes: + - application/json + parameters: + - name: User + in: body + required: true + schema: + $ref: "#/definitions/UserRequest" + responses: + 400: + description: User creation failed + 201: + description: User created + schema: + $ref: "#/definitions/User" + + /users/{user_id}: + parameters: + - $ref: "#/parameters/user_id" + get: + tags: + - user + summary: Fetches a user profile + responses: + 200: + description: User profile + schema: + $ref: "#/definitions/User" + put: + tags: + - user + summary: Updates user details + consumes: + - application/json + parameters: + - name: User + in: body + required: true + schema: + $ref: "#/definitions/UserRequest" + responses: + 204: + description: User Updated + + delete: + tags: + - user + summary: Deletes user + responses: + 204: + description: User deleted + + /users/{user_id}/password: + parameters: + - $ref: "#/parameters/user_id" + post: + tags: + - user + summary: Updates user password + consumes: + - application/json + parameters: + - name: Password + in: body + required: true + schema: + type: object + properties: + password: + type: string + format: password + responses: + 204: + description: Password updated + # Projects /projects: get: @@ -416,15 +671,18 @@ paths: tags: - projects summary: Create a new project + consumes: + - application/json parameters: - name: Project in: body required: true schema: - $ref: '#/definitions/Project' + $ref: '#/definitions/ProjectRequest' responses: 201: description: Created project + /events: get: summary: Get Events related to Semaphore and projects you are part of @@ -510,14 +768,16 @@ paths: in: query required: true type: string - format: name/username/email/admin + enum: [name, username, email, admin] description: sorting name + x-example: email - name: order in: query required: true type: string - format: asc/desc + enum: [asc, desc] description: ordering manner + x-example: desc responses: 200: description: Users @@ -538,7 +798,7 @@ paths: properties: user_id: type: integer - format: userID + minimum: 2 admin: type: boolean responses: @@ -583,24 +843,28 @@ paths: - project summary: Get access keys linked to project parameters: + # TODO - the space in this parameter name results in a dredd warning - name: Key type in: query required: false type: string - format: ssh/aws/gcloud/do + enum: [ssh, aws, gcloud, do] description: Filter by key type + x-example: ssh - name: sort in: query required: true type: string - format: name/type + enum: [name, type] description: sorting name + x-example: type - name: order in: query required: true type: string - format: asc/desc + enum: [asc, desc] description: ordering manner + x-example: asc responses: 200: description: Access Keys @@ -617,7 +881,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/AccessKey" + $ref: "#/definitions/AccessKeyRequest" responses: 204: description: Access Key created @@ -636,7 +900,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/AccessKey" + $ref: "#/definitions/AccessKeyRequest" responses: 204: description: Key updated @@ -663,13 +927,14 @@ paths: in: query required: true type: string - format: name/git_url/ssh_key + enum: [name, git_url, ssh_key] description: sorting name - name: order in: query required: true type: string format: asc/desc + enum: [asc, desc] description: ordering manner responses: 200: @@ -687,7 +952,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/Repository" + $ref: "#/definitions/RepositoryRequest" responses: 204: description: Repository created @@ -716,14 +981,14 @@ paths: in: query required: true type: string - format: name/type description: sorting name + enum: [name, type] - name: order in: query required: true type: string - format: asc/desc description: ordering manner + enum: [asc, desc] responses: 200: description: inventory @@ -740,7 +1005,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/Inventory" + $ref: "#/definitions/InventoryRequest" responses: 201: description: inventory created @@ -759,7 +1024,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/Inventory" + $ref: "#/definitions/InventoryRequest" responses: 204: description: Inventory updated @@ -786,12 +1051,14 @@ paths: type: string format: name description: sorting name + x-example: 'db-deploy' - name: order in: query required: true type: string format: asc/desc description: ordering manner + x-example: desc responses: 200: description: environment @@ -808,7 +1075,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/Environment" + $ref: "#/definitions/EnvironmentRequest" responses: 204: description: Environment created @@ -825,7 +1092,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/Environment" + $ref: "#/definitions/EnvironmentRequest" responses: 204: description: Environment Updated @@ -850,14 +1117,14 @@ paths: in: query required: true type: string - format: alias/playbook/ssh_key/inventory/environment/repository description: sorting name + enum: [alias, playbook, ssh_key, inventory, environment, repository] - name: order in: query required: true type: string - format: asc/desc description: ordering manner + enum: [asc, desc] responses: 200: description: template @@ -874,7 +1141,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/Template" + $ref: "#/definitions/TemplateRequest" responses: 201: description: template created @@ -893,7 +1160,7 @@ paths: in: body required: true schema: - $ref: "#/definitions/Template" + $ref: "#/definitions/TemplateRequest" responses: 204: description: template updated @@ -973,6 +1240,13 @@ paths: description: Task schema: $ref: "#/definitions/Task" + delete: + tags: + - project + summary: Deletes task (including output) + responses: + 204: + description: task deleted /project/{project_id}/tasks/{task_id}/output: parameters: - $ref: '#/parameters/project_id' @@ -988,95 +1262,3 @@ paths: type: array items: $ref: "#/definitions/TaskOutput" - /project/{project_id}/tasks/{task_id}: - parameters: - - $ref: '#/parameters/project_id' - - $ref: '#/parameters/task_id' - delete: - tags: - - project - summary: Deletes task (including output) - responses: - 204: - description: task deleted - - # users - /users: - get: - tags: - - user - summary: Fetches all users - responses: - 200: - description: Users - schema: - type: array - items: - $ref: "#/definitions/User" - post: - tags: - - user - summary: Creates a user - parameters: - - name: User - in: body - required: true - schema: - $ref: "#/definitions/User" - responses: - 201: - description: User created - schema: - $ref: "#/definitions/User" - /users/{user_id}: - parameters: - - $ref: "#/parameters/user_id" - get: - tags: - - user - summary: Fetches a user profile - responses: - 200: - description: User profile - schema: - $ref: "#/definitions/User" - put: - tags: - - user - summary: Updates user details - parameters: - - name: User - in: body - required: true - schema: - $ref: "#/definitions/User" - responses: - 204: - description: User Updated - delete: - tags: - - user - summary: Deletes user - responses: - 204: - description: User deleted - /users/{user_id}/password: - parameters: - - $ref: "#/parameters/user_id" - post: - tags: - - user - summary: Updates user password - parameters: - - name: Password - in: body - required: true - schema: - type: object - properties: - password: - type: string - format: password - responses: - 204: - description: Password updated diff --git a/api/api_test.go b/api/api_test.go new file mode 100644 index 00000000..de0319ec --- /dev/null +++ b/api/api_test.go @@ -0,0 +1,33 @@ +package api + +import ( + "testing" + "github.com/go-openapi/loads" + "github.com/go-openapi/spec" + "github.com/go-openapi/validate" + "github.com/go-openapi/strfmt" + "os" + "log" + ) + +// TestApi Validates the api description in the root meets the swagger/openapi spec +func TestApiSchemaValidation(t *testing.T) { + dir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + fpath := dir+"/../api-docs.yml" + print(fpath) + document, err := loads.Spec(fpath) + if err != nil { + t.Fatal(err) + } + spc := spec.ExpandOptions{RelativeBase: fpath} + document, err = document.Expanded(&spc) + if err != nil { + t.Fatal(err) + } + if err := validate.Spec(document, strfmt.Default); err != nil { + t.Fatal(err) + } +} diff --git a/api/projects/projects.go b/api/projects/projects.go index 0a0e362a..f0d868ac 100644 --- a/api/projects/projects.go +++ b/api/projects/projects.go @@ -7,6 +7,7 @@ import ( "github.com/castawaylabs/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" + "time" "github.com/ansible-semaphore/semaphore/util" ) @@ -49,9 +50,13 @@ func AddProject(w http.ResponseWriter, r *http.Request) { } desc := "Project Created" + oType := "Project" if err := (db.Event{ ProjectID: &body.ID, Description: &desc, + ObjectType: &oType, + ObjectID: &body.ID, + Created: db.GetParsedTime(time.Now()), }.Insert()); err != nil { panic(err) } diff --git a/api/projects/users.go b/api/projects/users.go index c76fded4..7c5275f8 100644 --- a/api/projects/users.go +++ b/api/projects/users.go @@ -83,6 +83,7 @@ func AddUser(w http.ResponseWriter, r *http.Request) { return } + //TODO - check if user already exists if _, err := db.Mysql.Exec("insert into project__user set user_id=?, project_id=?, `admin`=?", user.UserID, project.ID, user.Admin); err != nil { panic(err) } diff --git a/api/router.go b/api/router.go index 0d6084e3..b8998437 100644 --- a/api/router.go +++ b/api/router.go @@ -16,12 +16,21 @@ import ( var publicAssets = packr.NewBox("../web/public") +//JSONMiddleware ensures that all the routes respond with Json, this is added by default to all routes +func JSONMiddleware(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") +} + +//PlainTextMiddleware resets headers to Plain Text if needed +func PlainTextMiddleware(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "text/plain; charset=utf-8") +} // Route declares all routes func Route() mulekick.Router { - r := mulekick.New(mux.NewRouter(), mulekick.CorsMiddleware) + r := mulekick.New(mux.NewRouter(), mulekick.CorsMiddleware, JSONMiddleware) r.NotFoundHandler = http.HandlerFunc(servePublic) - r.Get("/api/ping", mulekick.PongHandler) + r.Get("/api/ping", PlainTextMiddleware, mulekick.PongHandler) // set up the namespace api := r.Group("/api") @@ -45,7 +54,7 @@ func Route() mulekick.Router { api.Get("/tokens", getAPITokens) api.Post("/tokens", createAPIToken) - api.Delete("/tokens/:token_id", expireAPIToken) + api.Delete("/tokens/{token_id}", expireAPIToken) }(api.Group("/user")) api.Get("/projects", projects.GetProjects) diff --git a/api/user.go b/api/user.go index 9e559e33..a2bad4c1 100644 --- a/api/user.go +++ b/api/user.go @@ -43,7 +43,7 @@ func createAPIToken(w http.ResponseWriter, r *http.Request) { token := db.APIToken{ ID: strings.ToLower(base64.URLEncoding.EncodeToString(tokenID)), - Created: time.Now(), + Created: db.GetParsedTime(time.Now()), UserID: user.ID, Expired: false, } diff --git a/api/users.go b/api/users.go index 9011e2a8..da4a6286 100644 --- a/api/users.go +++ b/api/users.go @@ -25,6 +25,7 @@ func getUsers(w http.ResponseWriter, r *http.Request) { func addUser(w http.ResponseWriter, r *http.Request) { var user db.User if err := mulekick.Bind(w, r, &user); err != nil { + w.WriteHeader(http.StatusBadRequest) return } @@ -35,7 +36,7 @@ func addUser(w http.ResponseWriter, r *http.Request) { return } - user.Created = time.Now() + user.Created = db.GetParsedTime(time.Now()) if err := db.Mysql.Insert(&user); err != nil { panic(err) diff --git a/db/AccessKey.go b/db/AccessKey.go index 0430590e..30058c6b 100644 --- a/db/AccessKey.go +++ b/db/AccessKey.go @@ -10,7 +10,7 @@ import ( type AccessKey struct { ID int `db:"id" json:"id"` Name string `db:"name" json:"name" binding:"required"` - // 'aws/do/gcloud/ssh', + // 'aws/do/gcloud/ssh' Type string `db:"type" json:"type" binding:"required"` ProjectID *int `db:"project_id" json:"project_id"` diff --git a/db/mysql.go b/db/mysql.go index 4e0f0d14..6697c038 100644 --- a/db/mysql.go +++ b/db/mysql.go @@ -6,6 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/util" _ "github.com/go-sql-driver/mysql" // imports mysql driver "gopkg.in/gorp.v1" + "time" log "github.com/Sirupsen/logrus" ) @@ -13,7 +14,22 @@ import ( // db.Connect must be called to set this up correctly var Mysql *gorp.DbMap -// Connect to MySQL and initialize the Mysql object +// DatabaseTimeFormat represents the format that dredd uses to validate the datetime. +// This is not the same as the raw value we pass to a new object so +// we need to use this to coerce raw values to meet the API standard +// /^\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\d{2}:\d{2}:\d{2}Z$/ +const DatabaseTimeFormat = "2006-01-02T15:04:05:99Z" + +// GetParsedTime returns the timestamp as it will retrieved from the database +// This allows us to create timestamp consistency on return values from create requests +func GetParsedTime(t time.Time) time.Time { + parsedTime, err := time.Parse(DatabaseTimeFormat,t.Format(DatabaseTimeFormat)) + if err != nil { + log.Error(err) + } + return parsedTime +} +// Connect ensures that the db is connected and mapped properly with gorp func Connect() error { db, err := connect() if err != nil { diff --git a/deployment/docker/Readme.md b/deployment/docker/Readme.md index 314ab55d..02a3549f 100644 --- a/deployment/docker/Readme.md +++ b/deployment/docker/Readme.md @@ -62,6 +62,13 @@ which contains the go toolchain and glibc in alpine. Because the test image links your local volume it expects that you have run `task deps` and `task compile` locally as necessary to make the application usable. +## CI + +This context is a proxyless stack used to test the API in the ci. Essentially it just installs the app, adds a few bootstrapping files +and starts up so that dredd can be run against it. This should not be used in production as it does not remove the build toolchain, +or source code. +It is more advisable to use the dev context locally as it volume links the application directory and defaults to the watch task. + ## Convenience Functions ### dc:dev diff --git a/deployment/docker/ci/Dockerfile b/deployment/docker/ci/Dockerfile new file mode 100644 index 00000000..d0f5fe00 --- /dev/null +++ b/deployment/docker/ci/Dockerfile @@ -0,0 +1,36 @@ +# Golang testing image with some tools already installed +FROM tomwhiston/micro-golang:test-base + +LABEL maintainer="Tom Whiston " + +ENV SEMAPHORE_VERSION="development" SEMAPHORE_ARCH="linux_amd64" \ + SEMAPHORE_CONFIG_PATH="${SEMAPHORE_CONFIG_PATH:-/etc/semaphore}" \ + APP_ROOT="/go/src/github.com/ansible-semaphore/semaphore/" + +# hadolint ignore=DL3013 +RUN apk add --no-cache git mysql-client python py-pip py-openssl openssl ca-certificates curl curl-dev openssh-client tini nodejs nodejs-npm bash rsync && \ + apk --update add --virtual build-dependencies python-dev libffi-dev openssl-dev build-base && \ + pip install --upgrade pip cffi && \ + pip install ansible && \ + apk del build-dependencies && \ + rm -rf /var/cache/apk/* && \ + adduser -D -u 1002 -g 0 semaphore && \ + mkdir -p /go/src/github.com/ansible-semaphore/semaphore && \ + mkdir -p /tmp/semaphore && \ + mkdir -p /etc/semaphore && \ + chown -R semaphore:0 /go && \ + chown -R semaphore:0 /tmp/semaphore && \ + chown -R semaphore:0 /etc/semaphore && \ + ssh-keygen -t rsa -q -f "/root/.ssh/id_rsa" -N "" && \ + ssh-keyscan -H github.com > /root/.ssh/known_hosts && \ + go get -u -v github.com/go-task/task/cmd/task + +# Copy in app source +WORKDIR ${APP_ROOT} +COPY . ${APP_ROOT} +RUN deployment/docker/ci/bin/install + +USER semaphore +EXPOSE 3000 +ENTRYPOINT ["/usr/local/bin/semaphore-wrapper"] +CMD ["./bin/semaphore", "--config", "/etc/semaphore/config.json"] \ No newline at end of file diff --git a/deployment/docker/ci/bin/install b/deployment/docker/ci/bin/install new file mode 100755 index 00000000..c36f6d0e --- /dev/null +++ b/deployment/docker/ci/bin/install @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + + +echo "--> Turn off StrictKeyChecking" +cat > /etc/ssh/ssh_config < Install Semaphore entrypoint wrapper script" +cp ./deployment/docker/common/semaphore-wrapper /usr/local/bin/semaphore-wrapper +task deps +task compile +task build:local diff --git a/deployment/docker/ci/docker-compose.yml b/deployment/docker/ci/docker-compose.yml new file mode 100644 index 00000000..b2fdca45 --- /dev/null +++ b/deployment/docker/ci/docker-compose.yml @@ -0,0 +1,45 @@ +version: '2' + +services: + mysql: + image: mysql + environment: + MYSQL_RANDOM_ROOT_PASSWORD: 'yes' + MYSQL_DATABASE: semaphore + MYSQL_USER: semaphore + MYSQL_PASSWORD: semaphore + ports: + - "3306:3306" + + semaphore_ci: + image: ansiblesemaphore/semaphore:ci-compose + build: + context: ./../../../ + dockerfile: ./deployment/docker/ci/Dockerfile + environment: + SEMAPHORE_DB_USER: semaphore + SEMAPHORE_DB_PASS: semaphore + SEMAPHORE_DB_HOST: mysql + SEMAPHORE_DB_PORT: 3306 + SEMAPHORE_DB: semaphore + SEMAPHORE_PLAYBOOK_PATH: /etc/semaphore + SEMAPHORE_ADMIN_PASSWORD: password + SEMAPHORE_ADMIN_NAME: "Developer" + SEMAPHORE_ADMIN_EMAIL: admin@localhost + SEMAPHORE_ADMIN: admin + SEMAPHORE_WEB_ROOT: http://0.0.0.0:3000 + ports: + - "3000:3000" + depends_on: + - mysql + + dredd: + image: ansiblesemaphore/dredd:ci + command: ["--config", ".dredd/dredd.yml"] + build: + context: ./../../../ + dockerfile: ./deployment/docker/ci/dredd.Dockerfile + depends_on: + - semaphore_ci + - mysql + diff --git a/deployment/docker/ci/dredd.Dockerfile b/deployment/docker/ci/dredd.Dockerfile new file mode 100644 index 00000000..7678cd10 --- /dev/null +++ b/deployment/docker/ci/dredd.Dockerfile @@ -0,0 +1,24 @@ +# hadolint ignore=DL3006 +FROM tomwhiston/dredd:latest + +ENV TASK_VERSION=v2.0.1 \ + GOPATH=/home/developer/go \ + SEMAPHORE_SERVICE=semaphore_ci \ + SEMAPHORE_PORT=3000 \ + MYSQL_SERVICE=mysql \ + MYSQL_PORT=3306 + +# We need the source and task to compile the hooks +USER 0 +RUN dnf install -y nc +COPY deployment/docker/ci/dredd/entrypoint /usr/local/bin +COPY . /home/developer/go/src/github.com/ansible-semaphore/semaphore +WORKDIR /usr/local/bin +RUN curl -L "https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_amd64.tar.gz" | tar xvz && \ + chown -R developer /home/developer/go + +# Get tools and do compile +WORKDIR /home/developer/go/src/github.com/ansible-semaphore/semaphore +RUN task deps:tools && task deps:be && task compile:be && task compile:api:hooks + +ENTRYPOINT ["/usr/local/bin/entrypoint"] \ No newline at end of file diff --git a/deployment/docker/ci/dredd/entrypoint b/deployment/docker/ci/dredd/entrypoint new file mode 100755 index 00000000..fd9a9484 --- /dev/null +++ b/deployment/docker/ci/dredd/entrypoint @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -e + +echo "---> Add Config" +cat > ./.dredd/config.json < Waiting for semaphore" +while ! nc -z ${SEMAPHORE_SERVICE} ${SEMAPHORE_PORT}; do + sleep 1 +done + +echo "---> Run Dredd" +# We do this because otherwise it can fail out +sleep 5 +/home/developer/node_modules/.bin/dredd $@ \ No newline at end of file diff --git a/deployment/docker/dev/Dockerfile b/deployment/docker/dev/Dockerfile index 837ad4bd..c5128769 100644 --- a/deployment/docker/dev/Dockerfile +++ b/deployment/docker/dev/Dockerfile @@ -7,7 +7,7 @@ ENV SEMAPHORE_VERSION="development" SEMAPHORE_ARCH="linux_amd64" \ SEMAPHORE_CONFIG_PATH="${SEMAPHORE_CONFIG_PATH:-/etc/semaphore}" \ APP_ROOT="/go/src/github.com/ansible-semaphore/semaphore/" -RUN apk add --no-cache git mysql-client python py-pip py-openssl openssl ca-certificates curl openssh-client tini nodejs nodejs-npm bash rsync && \ +RUN apk add --no-cache git mysql-client python py-pip py-openssl openssl ca-certificates curl curl-dev openssh-client tini nodejs nodejs-npm bash rsync && \ apk --update add --virtual build-dependencies python-dev libffi-dev openssl-dev build-base && \ pip install --upgrade pip cffi && \ pip install ansible && \ diff --git a/deployment/docker/dev/bin/install b/deployment/docker/dev/bin/install index 8ce0ca2c..3c438b97 100755 --- a/deployment/docker/dev/bin/install +++ b/deployment/docker/dev/bin/install @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + echo "--> Turn off StrictKeyChecking" cat > /etc/ssh/ssh_config <