circleci workflows and docker image build and push

.circleci/config.yml

@@ -1,93 +1,253 @@
-# This configuration was automatically generated from a CircleCI 1.0 config.
+# Workflow based configuration
 version: 2
+
+aliases:
+
+  - &golang-image
+    image: circleci/golang:1.10
+
+  - &working-dir
+    /go/src/github.com/ansible-semaphore/semaphore
+
+  - &store-bin-artifacts
+    store_artifacts:
+      path: /go/src/github.com/ansible-semaphore/semaphore/bin
+
+  - &install-task-binary
+    run:
+      name: install task binary
+      command: |
+        cd /go/bin
+        curl -L https://github.com/go-task/task/releases/download/v2.0.1/task_linux_amd64.tar.gz | tar xvz
+        cd -
+
+  - &persist-bin
+    persist_to_workspace:
+      root: .
+      paths:
+        - bin/*
+
+  - &install-node
+    run:
+      name: Install node
+      command: |
+        set +e
+        curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.5/install.sh | bash
+        export NVM_DIR="$HOME/.nvm"
+        [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+        nvm install 8.2.0 && nvm alias default 8.2.0
+        # Each step uses the same `$BASH_ENV`, so need to modify it
+        echo 'export NVM_DIR="$HOME/.nvm"' >> $BASH_ENV
+        echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV
+
+  - &test-compile-changes
+    run:
+      name: test that compile did not create/modify untracked files
+      command: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json'
+
+  - &save-npm-cache
+    save_cache:
+      key: v1-npm-deps-{{ checksum "web/package.json" }}
+      paths:
+        - web/node_modules
+
+  - &save-go-cache
+    save_cache:
+      key: v1-go-deps-{{ checksum "Gopkg.lock" }}-{{ checksum "Gopkg.toml" }}
+      paths:
+        - vendor
+
+  - &load-npm-cache
+    restore_cache:
+      keys:
+        - v1-npm-deps-{{ checksum "web/package.json" }}
+        - v1-npm-deps-
+
+  - &load-go-cache
+    restore_cache:
+      keys:
+        - v1-go-deps-{{ checksum "Gopkg.lock" }}-{{ checksum "Gopkg.toml" }}
+        - v1-go-deps-{{ checksum "Gopkg.lock" }}
+        - v1-go-deps-
+
 jobs:
-  build:
-    machine: true
-    # we hard code this because even in a fork our code references the canonical import urls
-    working_directory: ~/.go_workspace/src/github.com/ansible-semaphore/semaphore
+  build:local:
+    docker:
+      - *golang-image
+    working_directory: *working-dir
     steps:
-    # Remove go and restore the dependency cache
-    - run: sudo rm -rf /usr/local/go
-    - run: mkdir -p ${GOPATH}/bin
-    - restore_cache:
-        keys:
-        # This branch if available
-        - v4-dep-{{ .Branch }}-
-        # Default branch if not
-        - v4-dep-develop-
-        # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
-        - v4-dep-
-    - run: sudo apt-get update
-    - run: sudo apt-get install rpm
-    # Install node and go
-    - run:
-        name: Install node
-        command: |
-          set +e
-          curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.5/install.sh | bash
-          export NVM_DIR="/opt/circleci/.nvm"
-          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-          nvm install 8.2.0 && nvm alias default 8.2.0
-          # Each step uses the same `$BASH_ENV`, so need to modify it
-          echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV
-          echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV
-    - run: sudo curl -L https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz > /tmp/go.tar.gz
-    - run: sudo tar -C /usr/local -xzf /tmp/go.tar.gz
-    # Needed for task orchestration
-    - run: go get -u -v github.com/go-task/task/cmd/task
-    # Needed only in ci to post coverage reports
-    - run: go get github.com/schrej/godacov
-    # Get app and install all its deps
-    - checkout
-    # Circle can provide a concatenated gopath and the second item is not actually in the path
-    # which will cause builds to fail if deps are placed there
-    - run: GOPATH=/home/circleci/.go_workspace task deps
-    - run: task deps:docker
-    # Save dependency cache
-    - save_cache:
-        key: v4-dep-{{ .Branch }}-{{ epoch }}
-        paths:
-        - ~/.go_workspace/src/github.com/ansible-semaphore/semaphore
-        - ~/.go_workspace/bin
-        - /opt/circleci/.nvm
-    # Post cache compile
-    - run: task compile
-    # Test
-    # This looks like utter filth in circleci v2 but we have no choice apart from this escaping madness
-    - run: "cat > config.json <<EOF\n{\n\t\"mysql\": {\n\t\t\"host\": \"0.0.0.0:3306\"\
-        ,\n\t\t\"user\": \"root\",\n\t\t\"pass\": \"\",\n\t\t\"name\": \"circle_test\"\
-        \n\t},\n\t\"email_alert\": false\n}\nEOF\n"
-    - run: |
-           docker run -d -p "3306:3306" --health-cmd='mysqladmin ping --silent' -e MYSQL_ALLOW_EMPTY_PASSWORD=true -e MYSQL_DATABASE=circle_test --name=mysql mysql:5.6
-           function getContainerHealth {
-             docker inspect --format "{{json .State.Health.Status }}" $1
-           }
+      - run: export
+      - *install-node
+      - *install-task-binary
+      - checkout
+      - *load-go-cache
+      - *load-npm-cache
+      - run: task deps
+      - *save-go-cache
+      - *save-npm-cache
+      - run: task compile
+      - *test-compile-changes
+      - run: task build:local
+      - *store-bin-artifacts
+      - *persist-bin
 
-           function waitContainer {
-             while STATUS=$(getContainerHealth $1); [ $STATUS != "\"healthy\"" ]; do
-               if [ $STATUS == "\"unhealthy\"" ]; then
-                 exit -1
-               fi
-               printf .
-               sleep 1
-             done
-           }
+  build:
+    docker:
+      - *golang-image
+    working_directory: *working-dir
+    steps:
+      - *install-node
+      - *install-task-binary
+      - run: sudo apt-get install rpm
+      - checkout
+      - *load-go-cache
+      - *load-npm-cache
+      - run: task deps
+      - run: task compile
+      - *test-compile-changes
+      - run: task build
+      - *store-bin-artifacts
 
-           waitContainer mysql
-    - run: go run cli/main.go --migrate -config config.json
-    - run: docker stop mysql && docker rm mysql
-    - run: task test
-    - run: context=prod task docker:test
-    # Build artifacts for all architectures
-    - run: task build
-    # Post coverage
-    - run: godacov -t "${CODACY_TOKEN}" -r ./coverage.out -c "${CIRCLE_SHA1}" || true
-    # Teardown
-    # Save test results
-    - store_test_results:
-        path: ~/.go_workspace/src/github.com/ansible-semaphore/semaphore/coverage.out
-    # Save artifacts
-    - store_artifacts:
-        path: ~/.go_workspace/src/github.com/ansible-semaphore/semaphore/bin
-    - store_artifacts:
-        path: ~/.go_workspace/src/github.com/ansible-semaphore/semaphore/coverage.out
+  # Run goverage and post results
+  test:golang:
+    docker:
+      - *golang-image
+    working_directory: *working-dir
+    steps:
+      - *install-task-binary
+      # Needed only in ci to post coverage reports
+      - run: go get github.com/schrej/godacov
+      - checkout
+      - *load-go-cache
+      - run: task deps:tools
+      - run: task deps:be
+      - run: task compile:be
+      - run: task test
+      - run: godacov -t "${CODACY_TOKEN}" -r ./coverage.out -c "${CIRCLE_SHA1}" || true
+      - store_test_results:
+          path: /go/src/github.com/ansible-semaphore/semaphore/coverage.out
+      - store_artifacts:
+          path: /go/src/github.com/ansible-semaphore/semaphore/coverage.out
+
+  test:integration:
+    docker:
+      - *golang-image
+      - image: circleci/mysql
+    working_directory: *working-dir
+    steps:
+      - attach_workspace:
+          at: *working-dir
+      # This looks like utter filth in circleci v2 but we have no choice apart from this escaping madness
+      - run: "cat > config.json <<EOF\n{\n\t\"mysql\": {\n\t\t\"host\": \"127.0.0.1:3306\"\
+          ,\n\t\t\"user\": \"root\",\n\t\t\"pass\": \"\",\n\t\t\"name\": \"circle_test\"\
+          \n\t},\n\t\"email_alert\": false\n}\nEOF\n"
+      - run:
+          name: Wait for db
+          command: dockerize -wait tcp://127.0.0.1:3306 -timeout 1m
+      - run: bin/semaphore --migrate -config config.json
+      # TODO - Here we could do some api/functional testing
+
+  test:docker:
+    docker:
+      - *golang-image
+    steps:
+      - *install-task-binary
+      - checkout
+      - setup_remote_docker
+      - run: context=prod task docker:test
+
+  deploy:dev:
+    docker:
+      - *golang-image
+    steps:
+      - *install-task-binary
+      - checkout
+      - setup_remote_docker
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: context=prod tag=develop task docker:build
+      - run: tag=develop task docker:push
+
+  deploy:prod:
+    docker:
+      - *golang-image
+    steps:
+      - *install-task-binary
+      - checkout
+      - setup_remote_docker
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: context=prod tag=latest task docker:build
+      - run: tag=latest task docker:push
+      - docker tag ansiblesemaphore/semaphore:latest ansiblesemaphore/semaphore:$CIRCLE_TAG
+      - run: tag=$CIRCLE_TAG task docker:push
+
+  release:
+    docker:
+      - *golang-image
+    working_directory: *working-dir
+    steps:
+      - *install-node
+      - *install-task-binary
+      - run: sudo apt-get install rpm
+      - checkout
+      - *load-go-cache
+      - *load-npm-cache
+      - run: task deps
+      - run: task compile
+      - *test-compile-changes
+      - run: task release
+      - *store-bin-artifacts
+
+
+workflows:
+  version: 2
+  build-deploy:
+    jobs:
+      - test:docker
+      - test:golang
+      - build:local
+      - test:integration:
+          requires:
+            - build:local
+
+# Don't build on master because build is just a gorelease without the release
+      - build:
+          requires:
+            - test:golang
+            - test:integration
+          filters:
+            branches:
+              ignore: master
+
+# Dev deploys require all tests to pass and app builds
+      - deploy:dev:
+          requires:
+            - build
+            - test:docker
+          filters:
+            branches:
+              only: develop
+
+# Production deploys only happen if everything passes
+# and we have a tag starting with v
+      - release:
+          requires:
+            - test:golang
+            - test:integration
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /^v.*/
+
+# This deploy also happens on develop so we can publish beta/rc images
+      - deploy:prod:
+          requires:
+            - test:docker
+            - release
+          filters:
+            branches:
+              only:
+                - master
+                - develop
+            tags:
+              only: /^v.*/

.gitignore

@@ -9,7 +9,8 @@ config.json
 node_modules/
 
 .idea/
-bin/
+/bin/
+
 *-packr.go
 util/version.go
 /vendor/

CONTRIBUTING.md

@@ -34,7 +34,6 @@ git clone --recursive git@github.com:ansible-semaphore/semaphore.git && cd semap
 3) Install dev dependencies
 
 ```
-go get ./... 
 go get -u github.com/go-task/task/cmd/task
 task deps
 ```

README.md

@@ -6,6 +6,7 @@
 [![semaphore on discord](https://img.shields.io/badge/discord-semaphore%20community-738bd7.svg)](https://discord.gg/ZW7Qu6a)
 
 - [Releases](https://github.com/ansible-semaphore/semaphore/releases)
+- [Docker Hub](https://hub.docker.com/r/ansiblesemaphore/semaphore/)
 - [Install Instructions](https://github.com/ansible-semaphore/semaphore/wiki/Installation)
 - [Troubleshooting](https://github.com/ansible-semaphore/semaphore/wiki/Troubleshooting)
 - [Contribution Guide](https://github.com/ansible-semaphore/semaphore/blob/develop/CONTRIBUTING.md)

Taskfile.yml

@@ -5,6 +5,10 @@
 # internally by other tasks and therefore are not listed when running `task` or `task -l`
 version: '2'
 
+vars:
+  docker_namespace: ansiblesemaphore
+  docker_image: semaphore
+
 tasks:
   all:
     desc: Install, Compile, Test and Build Semaphore for local architecture
@@ -86,7 +90,8 @@ tasks:
       BRANCH:
         sh: git rev-parse --abbrev-ref HEAD
       DIRTY:
-        sh: git status --porcelain
+        # We must exclude the package-lock file as npm install can change it!
+        sh: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json'
       SHA:
         sh: git log --pretty=format:'%h' -n 1
       TIMESTAMP:
@@ -214,7 +219,7 @@ tasks:
           context: "{{ .context }}"
           action: build
           tag: "{{ .tag }}"
-          args: -t castawaylabs/semaphore:{{ .tag }} .
+          args: -t "{{ .docker_namespace }}/{{ .docker_image }}:{{ .tag }}" .
 
   deps:docker:
     desc: Install docker testing dependencies. These must be installed explicitly and are not included in the general deps task.
@@ -241,13 +246,13 @@ tasks:
         vars:
           tag: "{{ .context }}-test"
       - task: docker:goss
-      - docker rmi "castawaylabs/semaphore:{{ .context }}-test"
+      - docker rmi "{{ .docker_namespace }}/{{ .docker_image }}:{{ .context }}-test"
 
   docker:goss:
     dir: "deployment/docker/{{ .context}}"
     deps: ['deps:docker']
     cmds:
-      - GOSS_FILES_STRATEGY='cp' dgoss run -it "castawaylabs/semaphore:{{ .context }}-test"
+      - GOSS_FILES_STRATEGY='cp' dgoss run -it "{{ .docker_namespace }}/{{ .docker_image }}:{{ .context }}-test"
 
   docker:lint:
     desc: hadolint a dockerfile. Ignores version pinning warning
@@ -255,6 +260,11 @@ tasks:
     cmds:
       - hadolint Dockerfile --ignore DL3018
 
+  docker:push:
+    desc: push a docker image to a repo. Defaults to the official docker hub
+    cmds:
+      - docker push {{ .docker_namespace }}/{{ .docker_image }}:{{ .tag }}
+
   # templated command to reduce code duplication
   docker:
     vars:

deployment/docker/Readme.md

@@ -1,6 +1,6 @@
 # Docker Deployments
 
-Production images for each tag, latest and the development branch will be pushed to [docker hub](https://hub.docker.com/r/castawaylabs/semaphore).
+Production images for each tag, latest and the development branch will be pushed to [docker hub](https://hub.docker.com/r/ansiblesemaphore/semaphore).
 To build images locally see the contexts included here and use the `d` and `dc` tasks in the root Taskfile.yml to help with building and running.
 
 ## Contexts
@@ -11,7 +11,7 @@ To build a production image you should run
 
     context=prod task docker:build
     
-this will create an image called `castawaylabs/semaphore:latest` which will be compiled from the currently checked out code
+this will create an image called `ansiblesemaphore/semaphore:latest` which will be compiled from the currently checked out code
 
 This image is run as non root user 1001 (for PaaS systems such as openshift) and is build on alpine with added glibc.
 With ansible etc... installed in the container it is ~283MiB in size.

deployment/docker/dev/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       MYSQL_PASSWORD: semaphore
 
   semaphore_dev:
-    image: castawaylabs/semaphore-dev:latest
+    image: ansiblesemaphore/semaphore:dev-compose
     build:
       context: ./../../../
       dockerfile: ./deployment/docker/dev/Dockerfile

deployment/docker/prod/bin/install

@@ -8,7 +8,6 @@ export PATH="${PATH}:${GOBIN}"
 
 # Get prerequisites for building the app
 go get -u -v github.com/go-task/task/cmd/task
-go get ./...
 
 # Compile and build
 task deps

deployment/docker/prod/docker-compose.yml

@@ -13,7 +13,7 @@ services:
       MYSQL_PASSWORD: hx4hjxqthfwbfsy5535u4agfdscm
 
   semaphore:
-    image: castawaylabs/semaphore:prod
+    image: ansiblesemaphore/semaphore:prod-compose
     build:
       context: ./../../../
       dockerfile: ./deployment/docker/prod/Dockerfile
@@ -31,7 +31,7 @@ services:
     depends_on:
       - mysql
   semaphore_proxy:
-    image: castawaylabs/semaphore:proxy
+    image: ansiblesemaphore/proxy:latest
     build:
       context: ./proxy
     ports: