More data models and basic job runner

This commit is contained in:
Matej Kramny 2014-08-24 22:00:05 +01:00
parent 5465fb2e11
commit 7c466863f2
24 changed files with 403 additions and 18 deletions

1
.gitignore vendored
View File

@ -2,6 +2,7 @@ lib/credentials.js
lib/credentials.json
public/vendor
dist/
.vagrant
# Logs
logs

13
Vagrantfile vendored Normal file
View File

@ -0,0 +1,13 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
VAGRANTFILE_API_VERSION = "2"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "hashicorp/precise32"
config.vm.provider "virtualbox" do |vb|
# vb.gui = true
vb.customize ["modifyvm", :id, "--memory", "512"]
end
end

1
bin/semaphore.js Normal file
View File

@ -0,0 +1 @@
require('../lib/app');

0
bin/setup.js Normal file
View File

View File

@ -29,7 +29,7 @@ if (config.production) {
var releaseStage = config.production ? "production" : "development";
bugsnag.register("c0c7568710bb46d4bf14b3dad719dbbe", {
bugsnag.register(config.credentials.bugsnag_key, {
notifyReleaseStages: ["production"],
releaseStage: releaseStage
});

27
lib/models/Credential.js Normal file
View File

@ -0,0 +1,27 @@
var mongoose = require('mongoose')
var ObjectId = mongoose.Schema.ObjectId;
var schema = mongoose.Schema({
created: {
type: Date,
default: Date.now
},
credential_type: {
type: String,
enum: ['ssh', 'vault', 'git']
},
name: String,
password: String,
private_key: String,
public_key: String,
playbook: {
type: ObjectId,
ref: 'Playbook'
},
});
schema.index({
name: 1
});
module.exports = mongoose.model('Credential', schema);

26
lib/models/Host.js Normal file
View File

@ -0,0 +1,26 @@
var mongoose = require('mongoose')
var ObjectId = mongoose.Schema.ObjectId;
var schema = mongoose.Schema({
created: {
type: Date,
default: Date.now
},
name: String,
hostname: String,
group: {
type: ObjectId,
ref: 'HostGroup'
},
playbook: {
type: ObjectId,
ref: 'Playbook'
}
});
schema.index({
name: 1,
hostname: 1
});
module.exports = mongoose.model('Host', schema);

20
lib/models/HostGroup.js Normal file
View File

@ -0,0 +1,20 @@
var mongoose = require('mongoose')
var ObjectId = mongoose.Schema.ObjectId;
var schema = mongoose.Schema({
created: {
type: Date,
default: Date.now
},
name: String,
playbook: {
type: ObjectId,
ref: 'Playbook'
}
});
schema.index({
name: 1
});
module.exports = mongoose.model('HostGroup', schema);

22
lib/models/Job.js Normal file
View File

@ -0,0 +1,22 @@
var mongoose = require('mongoose')
var ObjectId = mongoose.Schema.ObjectId;
var schema = mongoose.Schema({
created: {
type: Date,
default: Date.now
},
playbook: {
type: ObjectId,
ref: 'Playbook'
},
name: String,
play_file: String, //x.yml
use_vault: Boolean
});
schema.index({
name: 1
});
module.exports = mongoose.model('Job', schema);

22
lib/models/Playbook.js Normal file
View File

@ -0,0 +1,22 @@
var mongoose = require('mongoose')
var ObjectId = mongoose.Schema.ObjectId;
var schema = mongoose.Schema({
created: {
type: Date,
default: Date.now
},
name: String,
location: String, // Git URL
vault_password: String,
credential: {
type: ObjectId,
ref: 'Credential'
}
});
schema.index({
name: 1
});
module.exports = mongoose.model('Playbook', schema);

23
lib/models/Task.js Normal file
View File

@ -0,0 +1,23 @@
var mongoose = require('mongoose')
var ObjectId = mongoose.Schema.ObjectId;
var schema = mongoose.Schema({
created: {
type: Date,
default: Date.now
},
job: {
type: ObjectId,
ref: 'Job'
},
status: {
type: String,
enum: ['Failed', 'Running', 'Queued']
}
});
schema.index({
status: 1
});
module.exports = mongoose.model('Task', schema);

View File

@ -1,4 +1,10 @@
var manifest = [
'Credential',
'Host',
'HostGroup',
'Job',
'Playbook',
'Task',
'User'
];

View File

@ -1,9 +0,0 @@
exports.httpRouter = function () {
}
exports.router = function () {
}

View File

@ -1,12 +1,13 @@
var util = require('../util')
, auth = require('./auth')
, apps = require('./apps')
, playbooks = require('./playbook/playbooks')
, profile = require('./profile')
exports.router = function(app) {
var templates = require('../templates')(app);
templates.route([
auth,
// apps
playbooks
]);
templates.add('homepage')
@ -16,7 +17,7 @@ exports.router = function(app) {
app.all('*', util.authorized);
// Handle HTTP reqs
apps.httpRouter(app);
playbooks.httpRouter(app);
// only json beyond this point
app.get('*', function(req, res, next) {
@ -31,7 +32,8 @@ exports.router = function(app) {
});
auth.router(app);
apps.router(app);
playbooks.router(app);
profile.router(app);
}
function layout (req, res) {

View File

@ -0,0 +1,24 @@
var models = require('../../models')
exports.unauthorized = function (app, template) {
template([
'add'
], {
prefix: 'playbook'
});
}
exports.httpRouter = function () {
}
exports.router = function (app) {
app.get('/playbooks', getPlaybooks)
}
function getPlaybooks (req, res) {
models.Playbook.find({
}, function (err, playbooks) {
res.send(playbooks)
})
}

13
lib/routes/profile.js Normal file
View File

@ -0,0 +1,13 @@
exports.router = function (app, template) {
app.get('/profile', getProfile)
}
function getProfile (req, res) {
res.send({
_id: req.user._id,
name: req.user.name,
username: req.user.username,
email: req.user.email
})
}

168
lib/runner.js Normal file
View File

@ -0,0 +1,168 @@
var async = require('async');
var config = require('./config');
var models = require('./models');
var fs = require('fs');
var spawn = require('child_process').spawn
exports.queue = async.queue(worker, 1);
function worker (task, callback) {
// Task is to be model Task
// Download the git project
// Set up hosts file
// Set up vault pwd file
// Set up private key
// Execute ansible-playbook -i hosts --ask-vault-pass --private-key=~/.ssh/ansible_key task.yml
async.waterfall([
function (done) {
task.populate('job', function (err) {
done(err, task);
})
},
function (task, done) {
models.Playbook.findOne({ _id: task.job.playbook }, function (err, playbook) {
done(err, task, playbook)
});
},
function (task, playbook, done) {
playbook.populate('credential', function (err) {
done(err, task, playbook)
});
},
installHostKeys,
pullGit,
setupHosts,
setupVault,
playTheBook,
function (task, playbook, done) {
var rmrf = spawn('rm', ['-rf', '/root/playbook_'+playbook._id])
rmrf.on('close', function () {
done(null, playbook)
})
}
], callback);
}
function installHostKeys (task, playbook, done) {
// Install the private key
var location = '/root/.ssh/id_rsa';
fs.mkdir('/root/.ssh', 448, function() {
async.parallel([
function (done) {
fs.writeFile(location, playbook.credential.private_key, {
mode: 384 // base 8 = 0600
}, done);
},
function (done) {
fs.writeFile(location+'.pub', playbook.credential.public_key, {
mode: 420 // base 8 = 0644
}, done);
},
function (done) {
var config = "Host *\n\
StrictHostKeyChecking no\n\
CheckHostIp no\n\
PasswordAuthentication no\n";
fs.writeFile('/root/.ssh/config', config, {
mode: 420 // 0644
}, done);
}
], function (err) {
done(err, task, playbook)
});
});
}
function pullGit (task, playbook, done) {
// Pull from git
var install = spawn(config.path+"/scripts/pullGit.sh", [playbook.location, 'playbook_'+playbook._id], {
cwd: '/root/',
env: {
HOME: '/root/',
OLDPWD: '/root/',
PWD: '/root/',
LOGNAME: 'root',
USER: 'root',
TERM: 'xterm',
SHELL: '/bin/bash',
PATH: '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin',
LANG: 'en_GB.UTF-8'
}
});
install.stdout.on('data', function (chunk) {
console.log('out', chunk.toString('utf8'))
});
install.stderr.on('data', function (chunk) {
console.log('err', chunk.toString('utf8'))
});
install.on('close', function(code) {
console.log('done.', code)
done(null, task, playbook);
});
}
function setupHosts (task, playbook, done) {
var hostfile = '';
models.HostGroup.find({
playbook: playbook._id
}, function (err, hostgroups) {
async.each(hostgroups, function (group, cb) {
models.Host.find({
group: group._id
}, function (err, hosts) {
hostfile += "["+group.name+"]\n";
for (var i = 0; i < hosts.length; i++) {
hostfile += hosts[i].hostname+"\n";
}
cb();
});
}, function () {
console.log(hostfile);
fs.writeFile('/root/playbook_'+playbook._id+'/semaphore_hosts', hostfile, function (err) {
done(err, task, playbook);
});
});
});
}
function setupVault (task, playbook, done) {
fs.writeFile('/root/playbook_'+playbook._id+'/semaphore_vault_pwd', playbook.vault_password, function (err) {
done(err, task, playbook);
})
}
function playTheBook (task, playbook, done) {
var playbook = spawn("ansible-playbook", ['-i', 'semaphore_hosts', '--vault-password-file='+'semaphore_vault_pwd', '--private-key=/root/.ssh/id_rsa', task.job.play_file], {
cwd: '/root/playbook_'+playbook._id,
env: {
HOME: '/root/',
OLDPWD: '/root/',
PWD: '/root/playbook_'+playbook._id,
LOGNAME: 'root',
USER: 'root',
TERM: 'xterm',
SHELL: '/bin/bash',
PATH: '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin',
LANG: 'en_GB.UTF-8'
}
});
playbook.stdout.on('data', function (chunk) {
console.log('out', chunk.toString('utf8'))
});
playbook.stderr.on('data', function (chunk) {
console.log('err', chunk.toString('utf8'))
});
playbook.on('close', function(code) {
console.log('done.', code)
done(null, task, playbook);
});
}

10
lib/scripts/pullGit.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
printf "#\041/bin/bash\nssh -i /root/.ssh/id_rsa \$1 \$2\n" > /root/ssh_wrapper.sh
chmod +x /root/ssh_wrapper.sh
cd /root
GIT_SSH=/root/ssh_wrapper.sh git clone "$1" $2
if [ $? -ne 0 ]; then
exit 2
fi

View File

@ -1,7 +1,7 @@
doctype
html
head
meta(http-equiv="Content-Type", content="text/html; charset=utf-8;")
meta(http-equiv="Content-Type" content="text/html; charset=utf-8;")
meta(name="viewport" content="width=device-width, initial-scale=1.0")
block meta
title(ng-bind-template="{{ pageTitle }} - Semaphore") Semaphore

View File

@ -1,7 +1,7 @@
doctype
html
head
meta(http-equiv="Content-Type", content="text/html; charset=utf-8;")
meta(http-equiv="Content-Type" content="text/html; charset=utf-8;")
meta(name="viewport" content="width=device-width, initial-scale=1.0")
block meta
title(ng-bind-template="{{ pageTitle }} - Semaphore") Semaphore
@ -19,6 +19,7 @@ html
.col-sm-3.col-lg-2
ul.nav
h2.text-center: a(ui-sref="homepage") Semaphore
button.btn.btn-block.btn-default(ui-sref="addPlaybook") Add Playbook
li(ng-repeat="playbook in playbooks")
a(ui-sref="playbook({ pid: playbook._id })") {{ playbook.name }}

View File

@ -0,0 +1 @@
h1 Add Playbook

View File

@ -0,0 +1,12 @@
define([
'app'
], function(app) {
app.config(function($stateProvider) {
$stateProvider
.state('addPlaybook', {
url: '/add',
pageTitle: 'Add Playbook',
templateUrl: "/view/playbook/add"
})
})
})

View File

@ -1,6 +1,7 @@
define([
'app',
'services/user'
'services/user',
'routes/playbooks'
], function(app) {
app.config(function($stateProvider, $urlRouterProvider, $locationProvider, $couchPotatoProvider) {
$locationProvider.html5Mode(true);
@ -29,7 +30,7 @@ define([
user.getUser(function() {})
$http.get('/playbooks').success(function(data, status) {
$rootScope.playbooks = data.playbooks;
$rootScope.playbooks = data;
})
})
})

1
ssh_vagrant.sh Executable file
View File

@ -0,0 +1 @@
vagrant ssh -- -R 27017:localhost:27017 -R 6379:localhost:6379