diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.default.yml similarity index 55% rename from .elasticbeanstalk/config.yml rename to .elasticbeanstalk/config.default.yml index 8e85b810c8cac3545fb18f80c00e243605f4df3c..cfbf204b095a7aef1f5628b0bec6ae3fa376e8a6 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.default.yml @@ -1,12 +1,9 @@ -branch-defaults: - $GIT_BRANCH: - environment: $EB_ENV +deploy: + artifact: deployment.zip global: - application_name: $EB_APP + application_name: App Service default_ec2_keyname: EastCoastBPKey - default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on 64bit Amazon Linux/2.8.1 + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on + 64bit Amazon Linux/2.8.1 default_region: us-east-1 - profile: null sc: git -deploy: - artifact: deployment.zip diff --git a/.gitignore b/.gitignore index ff3cf18f2baf551dd2a1a5f0a529cc81f7c91960..59cfb8ea33d3f15b1f06455a0baa1634ba28e8cd 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__ app/config/*.py !app/config/*.default.py config.json +.elasticbeanstalk/config.yml # PyCharm .idea diff --git a/Dockerfile b/Dockerfile index d607b8fe6bbc31ca7c7019d7813e7ebca0c71ede..6d4f8f3caf31b6f74906f9014103aa57437ab8b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,10 @@ FROM ubuntu:18.04 +ARG GITHUB_OAUTH_TOKEN +ARG DOMAIN +ARG NUM_PROCESSES +ARG NUM_THREADS + ENV CODEROOT=/home/docker/code # Log enironment variables. @@ -8,33 +13,26 @@ ENV NGINXREQ=/var/log/app/nginx.req.log ENV UWSGIERR=/var/log/app/uwsgi.err.log ENV UWSGIREQ=/var/log/app/uwsgi.req.log -# Custom environment variables. These change from project to project. -ARG DOMAIN -ENV DOMAIN=${DOMAIN} -ARG NUM_PROCESSES -ENV NUM_PROCESSES=${NUM_PROCESSES} -ARG NUM_THREADS -ENV NUM_THREADS=${NUM_THREADS} - # Install dependencies. -RUN apt-get update && apt-get install -y \ +RUN apt-get -qq update && apt-get install -y \ build-essential \ git \ libssl-dev \ libffi-dev \ nginx \ - python-dev \ + python3-dev \ python3 \ python3-pip \ supervisor \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install uwsgi +RUN pip3 install -q uwsgi COPY ./requirements.txt $CODEROOT/requirements.txt +RUN sed -i "s@git+ssh:\/\/git@git+https:\/\/$GITHUB_OAUTH_TOKEN@" $CODEROOT/requirements.txt # Install application requirements. -RUN pip3 install -r $CODEROOT/requirements.txt +RUN pip3 install -q -r $CODEROOT/requirements.txt # Put the code somewhere. ADD . $CODEROOT/ diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json index e8817f3dabd01777bf3981333327a1312bd34531..125ed54048b15436152b59f06517a74b4db404f7 100644 --- a/Dockerrun.aws.json +++ b/Dockerrun.aws.json @@ -1,9 +1,5 @@ { "AWSEBDockerrunVersion": "1", - "Authentication": { - "Bucket": "dockerauth.blocpower.org", - "Key": "$DOCKER_REPO.json" - }, "Image": { "Name": "blocp/$DOCKER_REPO" }, diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..027bba66fe3b3ae04679d1c4914915bea88e11d9 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,221 @@ +pipeline { + agent any + + environment { + PROJECT = '' // Value from EB_APP -- lowercase and without spaces + DEPLOY_TO = '' // User choice -- see parameters + COMMIT_HASH = '' // Short git commit hash + IMAGE = '' // "PROJECT:COMMIT_HASH" + VERSION = '' // Git version only applies to staging and prod + + EB_APP = '' // Value from .global.application_name in EB_CONFIG + EB_ENV = '' // "PROJECT-DEPLOY_TO" + EB_CONFIG = '.elasticbeanstalk/config.yml' + + NEVER_DEPLOY_ENV = 'none' + + GITHUB_CREDENTIALS = credentials('GITHUB_CREDENTIALS') + GITHUB_OAUTH_TOKEN = "${env.GITHUB_CREDENTIALS_PSW}" + } + + options { + buildDiscarder(logRotator(numToKeepStr: '60')) + } + + parameters { + choice( + name: 'DEPLOY_TO', + choices: 'none\ndev\nstaging\nprod', + description: 'Which environment do you want to deploy to?' + ) + string( + name: 'VERSION', + defaultValue: "none", + description: 'What version do you want to deploy? (Only for staging and prod)' + ) + } + + stages { + stage('Build preparations') { + steps { + sh "cp .elasticbeanstalk/config.default.yml $EB_CONFIG" + + script { + EB_APP = sh( + returnStdout: true, + script: """yq -r '.global.application_name' $EB_CONFIG""" + ).trim() + PROJECT = "${EB_APP.replaceAll("\\s", "").toLowerCase()}" + EB_ENV = "$PROJECT-${params.DEPLOY_TO}" + + if ((params.DEPLOY_TO == 'staging' && params.VERSION != 'none') || params.DEPLOY_TO == 'prod') { + git( + url: "https://${GITHUB_OAUTH_TOKEN}@github.com/Blocp/${PROJECT}.git", + branch: "${BRANCH_NAME}" + ) + gitCommitHash = sh(returnStdout: true, script: "git rev-list -n 1 ${params.VERSION} --").trim() + COMMIT_HASH = gitCommitHash.take(7) + + // set the build display name + currentBuild.displayName = "#${BUILD_ID}-${COMMIT_HASH}-${params.DEPLOY_TO}-${params.VERSION}" + } else { + gitCommitHash = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() + COMMIT_HASH = gitCommitHash.take(7) + + // set the build display name + currentBuild.displayName = "#${BUILD_ID}-${COMMIT_HASH}-${params.DEPLOY_TO}" + } + IMAGE = "$PROJECT:$COMMIT_HASH" + } + + echo "\n==================ENVIRONMENT VARS==================" + echo "Environment selected: ${params.DEPLOY_TO} -- Version: ${params.VERSION}" + echo "Current branch: ${env.GIT_BRANCH}" + echo "PROJECT: ${PROJECT} -- IMAGE: ${IMAGE}" + echo "EB_APP: ${EB_APP} -- EB_ENV: ${EB_ENV}" + echo "====================================================\n" + } + } + + stage('Build') { + when { + expression { params.DEPLOY_TO == 'dev' } + } + + environment { + ECR_CRED = 'ecr:us-east-1:AWS_CREDENTIALS' + + BUILD_ARGS = + "--build-arg DOMAIN=blocpower.io " + + "--build-arg NUM_PROCESSES=4 " + + "--build-arg NUM_THREADS=4 " + + "--build-arg GITHUB_OAUTH_TOKEN=$GITHUB_OAUTH_TOKEN" + } + + steps { + echo 'Building...' + + withCredentials([[ + $class: 'AmazonWebServicesCredentialsBinding', + credentialsId: 'AWS_CREDENTIALS', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' + ]]) { + + sh("eval \$(aws ecr get-login --no-include-email | sed 's|https://||')") + + script { + docker.build(IMAGE, "$BUILD_ARGS .") + docker.withRegistry("https://${env.ECR_REPO_HOST}", ECR_CRED) { + docker.image(IMAGE).push() + } + } + } + + echo 'Clean up images' + sh "docker rmi $IMAGE" + sh(""" docker rmi "${env.ECR_REPO_HOST}/$IMAGE" """) + } + } + + stage('Test') { + steps { + echo "Testing comming soon" + } + } + + stage('Deploy') { + when { + expression { params.DEPLOY_TO != NEVER_DEPLOY_ENV } + } + + steps { + echo "Starting deployment to $EB_ENV" + + echo "Updating EB dockerrun image name to ${env.ECR_REPO_HOST}/$IMAGE" + sh(""" + jq '.Image.Name = "${env.ECR_REPO_HOST}/$IMAGE"' Dockerrun.aws.json | \ + sponge Dockerrun.aws.json + """) + + sh 'zip -r deployment.zip Dockerrun.aws.json .ebextensions/*' + + withCredentials([[ + $class: 'AmazonWebServicesCredentialsBinding', + credentialsId: 'AWS_CREDENTIALS', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' + ]]) { + sh "eb use $EB_ENV" + sh "eb deploy $EB_ENV --label HASH:${COMMIT_HASH}-VERSION:${params.VERSION}" + } + + // Update image with environment deployed to + // build( + // job: 'Retag project in ECR', + // parameters: [ + // string(name: 'REPO_NAME', value: "$PROJECT"), + // string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), + // string(name: 'NEW_IMAGE_TAG', value: "${params.DEPLOY_TO}"), + // ] + // ) + + // script { + // if (params.VERSION != "none") { + // // Update image with version number + // build( + // job: 'Retag project in ECR', + // parameters: [ + // string(name: 'REPO_NAME', value: "$PROJECT"), + // string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), + // string(name: 'NEW_IMAGE_TAG', value: "${params.VERSION}"), + // ] + // ) + // } + // } + } + } + } + + post { + success { + script { + MESSAGE = "$PROJECT test ran successfully (commit hash: $COMMIT_HASH)" + + if (params.DEPLOY_TO != 'none') { + MESSAGE = "$PROJECT built to ${params.DEPLOY_TO} (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" + } + + slackSend( + color: 'good', + message: "$MESSAGE" + ) + + if (params.DEPLOY_TO == 'prod') { + slackSend( + color: 'good', + channel: '#dev', + message: "New version of $PROJECT (${params.VERSION}) in production (commit hash: $COMMIT_HASH)" + ) + } + } + } + + failure { + script { + MESSAGE = "$PROJECT test or build failed (commit hash: $COMMIT_HASH)" + + if (params.DEPLOY_TO == 'dev') { + MESSAGE = "$PROJECT build or test failed in ${params.DEPLOY_TO} (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" + } else { + MESSAGE = "$PROJECT failed to build to ${params.DEPLOY_TO} (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" + } + + slackSend( + color: 'danger', + message: "$MESSAGE" + ) + } + } + } +} diff --git a/app/__init__.py b/app/__init__.py index 0af6b2d19dcc4e4a2883198cc1257210c42aa466..c3708144230de5bf5331b679061cd37f84ea84db 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -27,7 +27,7 @@ def create_app(config): app.logger.addHandler(handler) - app.logger.info('Setting up application...') + app.logger.debug('Setting up application...') from . import views views.register(app) diff --git a/app/lib/service.py b/app/lib/service.py new file mode 100644 index 0000000000000000000000000000000000000000..bbfef367e6531f0ca002b9b64c7bda739858b60e --- /dev/null +++ b/app/lib/service.py @@ -0,0 +1,93 @@ +"""A library for inter-service communication.""" +import requests + + +class Service(object): + """A wrapper for requests to services.""" + # A default implementation of the cache. This is just a dictionary. + cache = {} + + def __init__(self, key, config, cache=None, cache_prefix=''): + """Sets information from config.""" + self.key = key + self.cache = cache or self.cache + self.cache_prefix = cache_prefix + self.config = config + try: + self.url = config['urls'][key] + except KeyError: + raise ValueError('No service found with key {}.'.format(key)) + + @property + def cache_key(self): + """The key to use when storing data in the cache.""" + return self.cache_prefix + self.key + + @property + def headers(self): + """Get a list of headers from the configuration and cache.""" + return { + self.config['headers']['app_key']: self.config['app_key'], + self.config['headers']['app_token']: self.cache.get(self.cache_key) or '', + self.config['headers']['app_secret']: self.config['app_secret'] + } + + def dispatch(self, method, url, *args, **kargs): + """Issue a request.""" + headers = {} + headers.update(self.headers) + headers.update(kargs.get('headers', {})) + kargs.update({'headers': headers}) + response = getattr(requests, method)(self.url + url, *args, **kargs) + self.cache.set( + self.cache_key, + response.headers.get(self.config['headers']['app_token'])) + return response + + def get(self, *args, **kargs): + """Issue a get request.""" + return self.dispatch('get', *args, **kargs) + + def post(self, url, *args, **kargs): + """Issue a post request.""" + return self.dispatch('post', *args, **kargs) + + def put(self, url, *args, **kargs): + """Issue a put request.""" + return self.dispatch('put', *args, **kargs) + + def delete(self, url, *args, **kargs): + """Issue a delete request.""" + return self.dispatch('delete', *args, **kargs) + + +class ServiceInterface(object): + """A wrapper for retrieving services from the current app.""" + def init_app(self, app): + """Set the application.""" + self.flask_app = app + + @property + def config(self): + """The configuration.""" + return self.flask_app.config.get('SERVICE_CONFIG') + + def __getattr__(self, key): + """Get the requested service object from app config.""" + from app.lib.red import redis + try: + return Service( + key, + self.config, + redis.server_apps, + self.config['app_key'] + '/') + except ValueError as e: + raise AttributeError(e) + + +services = ServiceInterface() + + +def register(app): + """Set the application on the interface.""" + services.init_app(app) diff --git a/app/lib/session.py b/app/lib/session.py new file mode 100644 index 0000000000000000000000000000000000000000..a26cebe8a439df417a241b2ea5ae15d933bfd786 --- /dev/null +++ b/app/lib/session.py @@ -0,0 +1,38 @@ +"""An in-memory session. + + This is associated with each request and provides a convenient way to pass + around data that would normally be associated with a cookie. Since our + application is stateless, this is not communicated with the frontend. There + is no actual cookie at use here. +""" +from flask.sessions import SessionMixin, SessionInterface + + +class NoCookieSession(dict, SessionMixin): + """A session to pass around.""" + pass + + +class NoCookieSessionInterface(SessionInterface): + """An interface to the no cookie session.""" + def open_session(self, app, request): + """Create a session from some headers. Ignore the cookie.""" + from app.lib.service import services + return NoCookieSession( + user_key=request.headers.get(app.config.get('HEADER_AUTH_KEY')), + user_token=request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), + app_key=request.headers.get(services.config['headers']['app_key']), + app_token=request.headers.get(services.config['headers']['app_token'])) + + def save_session(self, app, session, response): + """Respond with some headers from the session. Don't set the cookie.""" + from app.lib.service import services + response.headers.set(app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) + response.headers.set(app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) + response.headers.set(services.config['headers']['app_key'], session.get('app_key')) + response.headers.set(services.config['headers']['app_token'], session.get('app_token')) + + +def register(app): + """Set the session interface on the app.""" + app.session_interface = NoCookieSessionInterface()