From 0936a39778d466633598db7394f7bf05a8f8f396 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:04:43 -0500 Subject: [PATCH 01/37] Update to ubuntu 18 and python 3.6 --- Dockerfile | 21 +++++++++------------ requirements.txt | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5611b79..fab0f4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu +FROM ubuntu:18.04 ENV CODEROOT=/home/docker/code @@ -16,19 +16,16 @@ ENV NUM_PROCESSES=${NUM_PROCESSES} ARG NUM_THREADS ENV NUM_THREADS=${NUM_THREADS} -# Add the nginx reposoitory. We need 1.8 in order to support adding CORS headers to error responses. -RUN apt-get update -RUN apt-get install -y --no-install-recommends software-properties-common -RUN add-apt-repository ppa:nginx/stable - # Install dependencies. -RUN apt-get update -RUN apt-get install -y --no-install-recommends python3 python3-software-properties python3-dev python3-setuptools python3-pip -RUN apt-get install -y --no-install-recommends nginx supervisor -RUN apt-get install -y --no-install-recommends build-essential git -RUN apt-get install -y --no-install-recommends libpq-dev +RUN apt-get update && apt-get install -y \ + git \ + nginx \ + python3 \ + python3-pip \ + supervisor \ + && rm -rf /var/lib/apt/lists/* + RUN pip3 install uwsgi -RUN rm -rf /var/lib/apt/lists/* COPY ./requirements.txt $CODEROOT/requirements.txt diff --git a/requirements.txt b/requirements.txt index 7f1e1df..175ee5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ oauth2client==2.0.1 parse==1.6.6 pathspec==0.3.3 postgres==2.2.1 -psycopg2==2.6.1 +psycopg2==2.7.3.2 py==1.4.31 pyasn1==0.1.9 pyasn1-modules==0.0.8 -- GitLab From a70c625e9644f11724fe96a2270cd5171dc97925 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:23:03 -0500 Subject: [PATCH 02/37] Add python extras to dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index fab0f4d..1a65914 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,12 @@ ENV NUM_THREADS=${NUM_THREADS} # Install dependencies. RUN apt-get update && apt-get install -y \ + build-essential \ git \ + libssl-dev \ + libffi-dev \ nginx \ + python-dev \ python3 \ python3-pip \ supervisor \ -- GitLab From ecf43ac0d190107c118cd2773feead165f86b121 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:23:18 -0500 Subject: [PATCH 03/37] Update requirements to mactch other services --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 175ee5e..c76c3ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ arrow==0.7.0 blessed==1.9.5 +boto3==1.4.4 botocore==1.5.48 git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.0 cement==2.4.0 @@ -27,16 +28,16 @@ py==1.4.31 pyasn1==0.1.9 pyasn1-modules==0.0.8 pycparser==2.14 -python-dateutil==2.4.2 +python-dateutil==2.6.2 python-jose==1.3.2 PyYAML==3.11 redis==2.10.5 -requests==2.6.2 +requests==2.7 rsa==3.3 six==1.10.0 SQLAlchemy==1.0.13 texttable==0.8.4 wcwidth==0.1.6 websocket-client==0.35.0 -Werkzeug==0.11.4 +Werkzeug==0.11.5 WTForms==2.1 -- GitLab From d18c5621e0bcdac646799f909093a0035f7cc794 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:29:44 -0500 Subject: [PATCH 04/37] Fix python dateutil version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c76c3ab..a915976 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ py==1.4.31 pyasn1==0.1.9 pyasn1-modules==0.0.8 pycparser==2.14 -python-dateutil==2.6.2 +python-dateutil==2.5.3 python-jose==1.3.2 PyYAML==3.11 redis==2.10.5 -- GitLab From bcd0a7f175c2fe1ef4539c6752775f978e807300 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:32:24 -0500 Subject: [PATCH 05/37] Update six to 1.11.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a915976..cd5ea57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ PyYAML==3.11 redis==2.10.5 requests==2.7 rsa==3.3 -six==1.10.0 +six==1.11.0 SQLAlchemy==1.0.13 texttable==0.8.4 wcwidth==0.1.6 -- GitLab From cd6b7258c399d8f6ea255f62dfc68d0c05903e96 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 19:09:53 -0500 Subject: [PATCH 06/37] Add elasticbeanstalk logging --- Dockerfile | 10 +++++----- Dockerrun.aws.json | 3 ++- app/__init__.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a65914..d607b8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ FROM ubuntu:18.04 ENV CODEROOT=/home/docker/code # Log enironment variables. -ENV NGINXERR=/var/log/nginx.err.log -ENV NGINXREQ=/var/log/nginx.req.log -ENV UWSGIERR=/var/log/uwsgi.err.log -ENV UWSGIREQ=/var/log/uwsgi.req.log +ENV NGINXERR=/var/log/app/nginx.err.log +ENV NGINXREQ=/var/log/app/nginx.req.log +ENV UWSGIERR=/var/log/app/uwsgi.err.log +ENV UWSGIREQ=/var/log/app/uwsgi.req.log # Custom environment variables. These change from project to project. ARG DOMAIN @@ -49,7 +49,7 @@ RUN \ RUN useradd -ms /bin/bash www # Create the log files. -RUN \ +RUN mkdir /var/log/app/ && \ touch $NGINXERR && touch $NGINXREQ && \ touch $UWSGIERR && touch $UWSGIREQ diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json index d334d79..e8817f3 100644 --- a/Dockerrun.aws.json +++ b/Dockerrun.aws.json @@ -12,5 +12,6 @@ "HostPort": "80", "ContainerPort": "80" } - ] + ], + "Logging": "/var/log/app" } diff --git a/app/__init__.py b/app/__init__.py index dc103fa..3fae06a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -5,7 +5,7 @@ from flask import Flask MEGABYTE = 10**6 LOG_FORMAT = '[%(asctime)s] %(pathname)s:%(lineno)d %(levelname)s - %(message)s' -LOG_PATH = '/var/log/flask.log' +LOG_PATH = '/var/log/app/flask.log' def create_app(config): -- GitLab From 31efec99bcd11123009cad64fd5850be7b9f3e3e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Thu, 4 Jan 2018 12:48:05 -0500 Subject: [PATCH 07/37] Update default platform --- .elasticbeanstalk/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml index cd558c9..8e85b81 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.yml @@ -4,7 +4,7 @@ branch-defaults: global: application_name: $EB_APP default_ec2_keyname: EastCoastBPKey - default_platform: 64bit Amazon Linux 2015.09 v2.0.8 running Docker 1.9.1 + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on 64bit Amazon Linux/2.8.1 default_region: us-east-1 profile: null sc: git -- GitLab From 3f40a2e9548fb8d29ca501e448cd94b4b3c4932e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 01:22:34 -0500 Subject: [PATCH 08/37] Remove DebugHandler in development --- app/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 3fae06a..71715dc 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -15,12 +15,17 @@ def create_app(config): if config != 'config/local.py': handler = RotatingFileHandler(LOG_PATH, maxBytes=MEGABYTE, backupCount=1) - - handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt='%Y-%m-%d %H:%M:%S')) + if not app.debug: + handler.setLevel(logging.INFO) + app.logger.setLevel(logging.INFO) + else: + app.logger.handlers.pop() # remove DebugHandler + app.logger.setLevel(logging.DEBUG) + handler.setLevel(logging.DEBUG) + app.logger.addHandler(handler) - app.logger.setLevel(logging.INFO) app.logger.info('Setting up application...') -- GitLab From c7b61ddd33101903eacdb2a9d44edbc4dd4fbf2e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 01:31:27 -0500 Subject: [PATCH 09/37] Add cloudwatch log streams --- .ebextensions/log-streaming.config | 46 ++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .ebextensions/log-streaming.config diff --git a/.ebextensions/log-streaming.config b/.ebextensions/log-streaming.config new file mode 100644 index 0000000..6f51a58 --- /dev/null +++ b/.ebextensions/log-streaming.config @@ -0,0 +1,46 @@ +option_settings: + aws:elasticbeanstalk:cloudwatch:logs: + StreamLogs: true + DeleteOnTerminate: false + RetentionInDays: 180 + +packages: + yum: + awslogs: [] + +files: + "/etc/awslogs/config/logs.conf" : + mode: "000600" + owner: root + group: root + content: | + [/var/log/eb-docker/nginx.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.req.log + + [/var/log/eb-docker/nginx.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.err.log + + [/var/log/eb-docker/uwsgi.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.req.log + + [/var/log/eb-docker/uwsgi.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.err.log + + [/var/log/eb-docker/flask.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/flask.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/flask.log + +commands: + "01": + command: chkconfig awslogs on + "02": + command: service awslogs restart -- GitLab From 13cc7f20633cfc71f9e52544ad328296d7531f4f Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 01:32:03 -0500 Subject: [PATCH 10/37] Add elasticbeanstalk auto platform updater --- .ebextensions/managed-platform-update.config | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .ebextensions/managed-platform-update.config diff --git a/.ebextensions/managed-platform-update.config b/.ebextensions/managed-platform-update.config new file mode 100644 index 0000000..44bdf4a --- /dev/null +++ b/.ebextensions/managed-platform-update.config @@ -0,0 +1,7 @@ +option_settings: + aws:elasticbeanstalk:managedactions: + ManagedActionsEnabled: true + PreferredStartTime: "Mon:07:00" + aws:elasticbeanstalk:managedactions:platformupdate: + UpdateLevel: minor + InstanceRefreshEnabled: false -- GitLab From ed5f941d2a0264ab871e1038066f0d80eff3939e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:00:15 -0500 Subject: [PATCH 11/37] Add logging statements to app need --- app/permissions/application.py | 48 ++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/app/permissions/application.py b/app/permissions/application.py index b43f140..1775397 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -18,6 +18,7 @@ class AppNeed(Permission): # Check for a key. key = session.get('app_key') if not key: + current_app.logger.error('Session app key is empty') return False token = session.get('app_token') @@ -36,27 +37,33 @@ class AppNeed(Permission): # such, we can't use the normal service library for communication. secret = request.headers.get( services.config['headers']['app_secret'], None) - response = requests.get( - services.app.url + '/auth/', - params={ - 'client_key': key, - 'server_key': services.config['app_key']}, - headers={ - services.config['headers']['app_key']: key, - services.config['headers']['app_secret']: secret, - 'referer': request.headers.get('referer')}) + + try: + response = requests.get( + services.app.url + '/auth/', + params={ + 'client_key': key, + 'server_key': services.config['app_key'] + }, + headers={ + services.config['headers']['app_key']: key, + services.config['headers']['app_secret']: secret, + 'referer': request.headers.get('referer') + }) + response.raise_for_status() + except requests.exceptions.RequestException as e: + current_app.logger.error('Failed to communicate with appservice %s', e) + raise e # Check if the app authentication failed. if response.status_code != 200: + current_app.logger.error('App authentication failed') return False data = response.json() - if ( - 'data' not in data or - not isinstance(data['data'], list) or - len(data['data']) == 0 or - 'token' not in data['data'][0] - ): + if ('data' not in data or not isinstance(data['data'], list) + or len(data['data']) == 0 or 'token' not in data['data'][0]): + current_app.logger.error('Token not found in appservice response') return False token = data['data'][0]['token'] @@ -71,8 +78,8 @@ class AppNeed(Permission): # where there is already a token and it expires before we retrieve it. _, token = redis.client_apps.pipeline()\ .set(cache_key, token, - ex=current_app.config.get('APP_CACHE_EXPIRY'), - nx=True)\ + ex=current_app.config.get('APP_CACHE_EXPIRY'), + nx=True)\ .get(cache_key)\ .execute() @@ -80,16 +87,17 @@ class AppNeed(Permission): session['app_token'] = token return True + + app_need = AppNeed() + class RoleNeed(Permission): """Checks if a role is met by hitting the app service.""" @property def error(self): """Return an error based on the role.""" - return Unauthorized( - 'Please authenticate with an application with the {} role.'\ - .format(self.role)) + return Unauthorized(f'Please authenticate with an application with the {self.role} role.') def __init__(self, role): """Initialize with a role.""" -- GitLab From 3bbe7606f74fc63394843ec5bd086f0e31994063 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:01:17 -0500 Subject: [PATCH 12/37] Fix pylint and pep8 errors --- app/config/development.default.py | 2 +- app/config/local.default.py | 2 +- app/config/production.default.py | 2 +- app/config/staging.default.py | 2 +- app/config/test.default.py | 2 +- app/lib/auth0.py | 1 + app/lib/database.py | 6 ++++++ app/lib/exceptions.py | 1 + app/lib/service.py | 6 +++--- app/lib/session.py | 24 ++++++++---------------- app/models/base.py | 2 +- app/permissions/auth.py | 6 ++++-- app/views/__init__.py | 2 +- 13 files changed, 30 insertions(+), 28 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 216af3c..9b13b8b 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -22,7 +22,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/local.default.py b/app/config/local.default.py index a53dfc0..fef073e 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -19,7 +19,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/production.default.py b/app/config/production.default.py index d03bb1f..1d8a34d 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -22,7 +22,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/staging.default.py b/app/config/staging.default.py index 21bdbcb..d6b0bed 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -22,7 +22,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/test.default.py b/app/config/test.default.py index 6639a96..a187093 100644 --- a/app/config/test.default.py +++ b/app/config/test.default.py @@ -19,7 +19,7 @@ SERVICE_CONFIG = { } # App -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Test applications TEST_CLIENTS = { diff --git a/app/lib/auth0.py b/app/lib/auth0.py index 3014a0a..d6317a2 100644 --- a/app/lib/auth0.py +++ b/app/lib/auth0.py @@ -11,6 +11,7 @@ class FlaskAuth0: app.config.get('AUTH0_CLIENT_SECRET'), 'https://{}/api/v2/'.format(app.config.get('AUTH0_DOMAIN'))) + auth0_ = FlaskAuth0() diff --git a/app/lib/database.py b/app/lib/database.py index 9bb84f0..86e3658 100644 --- a/app/lib/database.py +++ b/app/lib/database.py @@ -7,11 +7,13 @@ from flask.ext.sqlalchemy import SQLAlchemy # and operations should be through this object. db = SQLAlchemy() + def sqlite_fk_pragma(dbapi_con, con_record): """Apply a one-time pragma to enforce foreign keys in SQLite.""" if isinstance(dbapi_con, sqlite3.Connection): dbapi_con.execute('pragma foreign_keys=ON') + def register(app): """Apply database hacks.""" db.init_app(app) @@ -19,6 +21,7 @@ def register(app): with app.app_context(): event.listen(db.engine, 'connect', sqlite_fk_pragma) + def commit(): """A wrapper for db.session.commit(). @@ -31,6 +34,7 @@ def commit(): db.session.rollback() raise e + class ProcTable: """ProcTable represents a simplified class of SQLAlchemy db.model""" @@ -42,12 +46,14 @@ class ProcTable: """List all columns""" return [column.key for column in self.columns] + class ProcColumn: """ProcColumn represents a column similar to SQLAlchemy column""" def __init__(self, key): self.key = key + def proc(model, method, limit=None, offset=None, **kwargs): """ Run stored procedure diff --git a/app/lib/exceptions.py b/app/lib/exceptions.py index 3a13142..b81bf82 100644 --- a/app/lib/exceptions.py +++ b/app/lib/exceptions.py @@ -2,6 +2,7 @@ from werkzeug.exceptions import ( default_exceptions, HTTPException, InternalServerError) from flask import jsonify + def register(app): """Wrap all exceptions so that they render to json.""" def jsonify_error(e): diff --git a/app/lib/service.py b/app/lib/service.py index d479e36..bbfef36 100644 --- a/app/lib/service.py +++ b/app/lib/service.py @@ -28,9 +28,9 @@ class Service(object): """Get a list of headers from the configuration and cache.""" return { self.config['headers']['app_key']: self.config['app_key'], - self.config['headers']['app_token']: \ - self.cache.get(self.cache_key) or '', - self.config['headers']['app_secret']: self.config['app_secret']} + self.config['headers']['app_token']: self.cache.get(self.cache_key) or '', + self.config['headers']['app_secret']: self.config['app_secret'] + } def dispatch(self, method, url, *args, **kargs): """Issue a request.""" diff --git a/app/lib/session.py b/app/lib/session.py index 7fb691c..a26cebe 100644 --- a/app/lib/session.py +++ b/app/lib/session.py @@ -19,26 +19,18 @@ class NoCookieSessionInterface(SessionInterface): """Create a session from some headers. Ignore the cookie.""" from app.lib.service import services return NoCookieSession( - user_key=\ - request.headers.get(app.config.get('HEADER_AUTH_KEY')), - user_token=\ - request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), - app_key=\ - request.headers.get(services.config['headers']['app_key']), - app_token=\ - request.headers.get(services.config['headers']['app_token'])) + user_key=request.headers.get(app.config.get('HEADER_AUTH_KEY')), + user_token=request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), + app_key=request.headers.get(services.config['headers']['app_key']), + app_token=request.headers.get(services.config['headers']['app_token'])) def save_session(self, app, session, response): """Respond with some headers from the session. Don't set the cookie.""" from app.lib.service import services - response.headers.set( - app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) - response.headers.set( - app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) - response.headers.set( - services.config['headers']['app_key'], session.get('app_key')) - response.headers.set( - services.config['headers']['app_token'], session.get('app_token')) + response.headers.set(app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) + response.headers.set(app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) + response.headers.set(services.config['headers']['app_key'], session.get('app_key')) + response.headers.set(services.config['headers']['app_token'], session.get('app_token')) def register(app): diff --git a/app/models/base.py b/app/models/base.py index bb0f150..31acd31 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -1,6 +1,6 @@ -import arrow import decimal import uuid +import arrow from sqlalchemy.sql import func from sqlalchemy.ext.declarative import declared_attr diff --git a/app/permissions/auth.py b/app/permissions/auth.py index b7be02c..299015d 100644 --- a/app/permissions/auth.py +++ b/app/permissions/auth.py @@ -1,7 +1,7 @@ """Permissions to check authentication.""" import json import requests -from flask import current_app, g +from flask import g from werkzeug.exceptions import Unauthorized from jose import jwt from .base import Permission @@ -15,8 +15,9 @@ class AuthNeed(Permission): super().__init__() def is_met(self): - from flask import session, request, current_app """Check for authentication with Auth0.""" + + from flask import session, request, current_app auth0_header = current_app.config.get('AUTH0_AUTH_HEADER') auth0_token = request.headers.get(auth0_header) @@ -61,5 +62,6 @@ class AuthNeed(Permission): return True return False + auth_need = AuthNeed() standard_login_need = app_need & auth_need diff --git a/app/views/__init__.py b/app/views/__init__.py index 1dcf967..015e00a 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -3,7 +3,7 @@ def register(app): """Register each view to the application. - + This can be used as a comprehensive list of all views that are present in the application. """ -- GitLab From c06076d42506bd219f6c6f030c83f1cdff7b9728 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:19:05 -0500 Subject: [PATCH 13/37] Update service urls to https --- app/config/development.default.py | 3 ++- app/config/local.default.py | 3 ++- app/config/production.default.py | 7 ++++--- app/config/staging.default.py | 7 ++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 9b13b8b..02edda6 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -14,7 +14,8 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { 'app': 'http://dev.appservice.blocpower.io', 'user': 'http://dev.userservice.blocpower.io', diff --git a/app/config/local.default.py b/app/config/local.default.py index fef073e..605fa43 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -11,7 +11,8 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { 'app': 'http://127.0.0.1:5400', 'user': 'http://127.0.0.1:5401', diff --git a/app/config/production.default.py b/app/config/production.default.py index 1d8a34d..2c17d09 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -14,10 +14,11 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { - 'app': 'http://app.s.blocpower.us', - 'user': 'http://user.s.blocpower.us', + 'app': 'https://appservice.blocpower.io', + 'user': 'https://userservice.blocpower.io', } } diff --git a/app/config/staging.default.py b/app/config/staging.default.py index d6b0bed..bc3fccc 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -14,10 +14,11 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { - 'app': 'http://staging.app.s.blocpower.us', - 'user': 'http://staging.user.s.blocpower.us' + 'app': 'https://staging.appservice.blocpower.io', + 'user': 'https://staging.userservice.blocpower.io' } } -- GitLab From b61918a0e38e83abade9c16fbdbc44ea115f24d6 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:39:35 -0500 Subject: [PATCH 14/37] Bump bpvalve to v1.3.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd5ea57..35b57ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ arrow==0.7.0 blessed==1.9.5 boto3==1.4.4 botocore==1.5.48 -git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.0 +git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.1 cement==2.4.0 colorama==0.3.3 docker-py==1.1.0 -- GitLab From 45f2af3e66fd69d53b7a07bf0ae247136e94ce28 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 16:00:10 -0500 Subject: [PATCH 15/37] Switch log leve in AppNedd --- app/permissions/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/permissions/application.py b/app/permissions/application.py index 1775397..42cc990 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -18,7 +18,7 @@ class AppNeed(Permission): # Check for a key. key = session.get('app_key') if not key: - current_app.logger.error('Session app key is empty') + current_app.logger.info('Session app key is empty') return False token = session.get('app_token') @@ -57,7 +57,7 @@ class AppNeed(Permission): # Check if the app authentication failed. if response.status_code != 200: - current_app.logger.error('App authentication failed') + current_app.logger.info('App authentication failed') return False data = response.json() -- GitLab From 5c8d9a9072abb4c6fad2294adbcfb3bd2fe1d816 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 16:14:53 -0500 Subject: [PATCH 16/37] Add https to dev config --- app/config/development.default.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 02edda6..d94b0b9 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -17,8 +17,8 @@ SERVICE_CONFIG = { 'app_secret': 'x-blocpower-app-secret' }, 'urls': { - 'app': 'http://dev.appservice.blocpower.io', - 'user': 'http://dev.userservice.blocpower.io', + 'app': 'https://dev.appservice.blocpower.io', + 'user': 'https://dev.userservice.blocpower.io', } } -- GitLab From 3b74567b8202c1a755bcf5e5dfb2b9f59ad6ba5c Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 9 Jan 2018 15:06:42 -0500 Subject: [PATCH 17/37] Change info log to debug --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 71715dc..743a630 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -27,7 +27,7 @@ def create_app(config): app.logger.addHandler(handler) - app.logger.info('Setting up application...') + app.logger.debug('Setting up application...') from . import views views.register(app) -- GitLab From e3aa5fed02a166d34e066ce63a58057b917bd632 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 9 Jan 2018 15:07:04 -0500 Subject: [PATCH 18/37] Add -qq flags for apt-get in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d607b8f..d28fbe2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ ARG NUM_THREADS ENV NUM_THREADS=${NUM_THREADS} # Install dependencies. -RUN apt-get update && apt-get install -y \ +RUN apt-get -qq update && apt-get -qq -y install \ build-essential \ git \ libssl-dev \ -- GitLab From 50778c68e4bb9646d3e5fce7bff23a136ec21d63 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Wed, 10 Jan 2018 19:14:54 -0500 Subject: [PATCH 19/37] Make pip output quiet --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d28fbe2..69effdf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,12 +29,12 @@ RUN apt-get -qq update && apt-get -qq -y install \ supervisor \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install uwsgi +RUN pip3 install -q uwsgi COPY ./requirements.txt $CODEROOT/requirements.txt # Install application requirements. -RUN pip3 install -r $CODEROOT/requirements.txt +RUN pip3 install -r -q $CODEROOT/requirements.txt # Put the code somewhere. ADD . $CODEROOT/ -- GitLab From af3015751ee7f23f5e189af022666c38adaee526 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Thu, 11 Jan 2018 13:49:09 -0500 Subject: [PATCH 20/37] Fix pip3 command --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 69effdf..b400bfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ RUN pip3 install -q uwsgi COPY ./requirements.txt $CODEROOT/requirements.txt # Install application requirements. -RUN pip3 install -r -q $CODEROOT/requirements.txt +RUN pip3 install -q -r $CODEROOT/requirements.txt # Put the code somewhere. ADD . $CODEROOT/ -- GitLab From 2a6e38b09d2e6336cd532219feea96f5c10baf7b Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 12 Jan 2018 15:36:41 -0500 Subject: [PATCH 21/37] Add config.bak.yml --- .elasticbeanstalk/config.bak.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .elasticbeanstalk/config.bak.yml diff --git a/.elasticbeanstalk/config.bak.yml b/.elasticbeanstalk/config.bak.yml new file mode 100644 index 0000000..ab62bbd --- /dev/null +++ b/.elasticbeanstalk/config.bak.yml @@ -0,0 +1,9 @@ +deploy: + artifact: deployment.zip +global: + application_name: $EB_APP + default_ec2_keyname: EastCoastBPKey + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on + 64bit Amazon Linux/2.8.1 + default_region: us-east-1 + sc: git -- GitLab From ac5d0718ce2296ed724ae2971bc4c28627788be9 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 14:57:11 -0500 Subject: [PATCH 22/37] Delete elasticbeanstalk folder --- .elasticbeanstalk/config.bak.yml | 9 --------- .elasticbeanstalk/config.yml | 12 ------------ 2 files changed, 21 deletions(-) delete mode 100644 .elasticbeanstalk/config.bak.yml delete mode 100644 .elasticbeanstalk/config.yml diff --git a/.elasticbeanstalk/config.bak.yml b/.elasticbeanstalk/config.bak.yml deleted file mode 100644 index ab62bbd..0000000 --- a/.elasticbeanstalk/config.bak.yml +++ /dev/null @@ -1,9 +0,0 @@ -deploy: - artifact: deployment.zip -global: - application_name: $EB_APP - default_ec2_keyname: EastCoastBPKey - default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on - 64bit Amazon Linux/2.8.1 - default_region: us-east-1 - sc: git diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml deleted file mode 100644 index 8e85b81..0000000 --- a/.elasticbeanstalk/config.yml +++ /dev/null @@ -1,12 +0,0 @@ -branch-defaults: - $GIT_BRANCH: - environment: $EB_ENV -global: - application_name: $EB_APP - default_ec2_keyname: EastCoastBPKey - default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on 64bit Amazon Linux/2.8.1 - default_region: us-east-1 - profile: null - sc: git -deploy: - artifact: deployment.zip -- GitLab From adde80aa7294243bd57ab57a2aedc1f56471ccea Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 15:01:00 -0500 Subject: [PATCH 23/37] Add Jenkinsfile --- Dockerrun.aws.json | 4 -- Jenkinsfile | 158 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 Jenkinsfile diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json index e8817f3..125ed54 100644 --- a/Dockerrun.aws.json +++ b/Dockerrun.aws.json @@ -1,9 +1,5 @@ { "AWSEBDockerrunVersion": "1", - "Authentication": { - "Bucket": "dockerauth.blocpower.org", - "Key": "$DOCKER_REPO.json" - }, "Image": { "Name": "blocp/$DOCKER_REPO" }, diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..efb23a4 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,158 @@ +pipeline { + agent any + + environment { + PROJECT = '' // Value from EB_APP -- lowercase and without spaces + DEPLOY_TO = '' // User choice -- see parameters + COMMIT_HASH = '' // Short git commit hash + IMAGE = '' // "PROJECT:COMMIT_HASH" + VERSION = '' // Git version only applies to staging and prod + + EB_APP = '' // Value from .global.application_name in EB_CONFIG + EB_ENV = '' // "PROJECT-DEPLOY_TO" + EB_CONFIG = '.elasticbeanstalk/config.yml' + + NEVER_DEPLOY_ENV = 'none' + + GITHUB_CREDENTIALS = credentials('GITHUB_CREDENTIALS') + GITHUB_OAUTH_TOKEN = "${env.GITHUB_CREDENTIALS_PSW}" + } + + parameters { + choice( + name: 'DEPLOY_TO', + choices: 'none\ndev\nstaging\nprod', + description: 'Which environment do you want to deploy to?' + ) + string( + name: 'VERSION', + defaultValue: "none", + description: 'What version do you want to deploy? (Only for staging and prod)' + ) + } + + stages { + stage('Build preparations') { + steps { + sh "cp .elasticbeanstalk/config.default.yml $EB_CONFIG" + + script { + EB_APP = sh( + returnStdout: true, + script: """yq -r '.global.application_name' $EB_CONFIG""" + ).trim() + PROJECT = "${EB_APP.replaceAll("\\s", "").toLowerCase()}" + EB_ENV = "$PROJECT-${params.DEPLOY_TO}" + + if (params.DEPLOY_TO == 'staging' || params.DEPLOY_TO == 'prod' || params.VERSION != 'none') { + git( + url: "https://${GITHUB_OAUTH_TOKEN}@github.com/Blocp/${PROJECT}.git", + branch: "${BRANCH_NAME}" + ) + gitCommitHash = sh(returnStdout: true, script: "git rev-list -n 1 ${params.VERSION} --").trim() + COMMIT_HASH = gitCommitHash.take(7) + + // set the build display name + currentBuild.displayName = "#${BUILD_ID}-${COMMIT_HASH}-${params.DEPLOY_TO}-${params.VERSION}" + } else { + gitCommitHash = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() + COMMIT_HASH = gitCommitHash.take(7) + + // set the build display name + currentBuild.displayName = "#${BUILD_ID}-${COMMIT_HASH}-${params.DEPLOY_TO}" + } + IMAGE = "$PROJECT:$COMMIT_HASH" + } + + echo "\n==================ENVIRONMENT VARS==================" + echo "Environment selected: ${params.DEPLOY_TO} -- Version: ${params.VERSION}" + echo "Current branch: ${env.GIT_BRANCH}" + echo "PROJECT: ${PROJECT} -- IMAGE: ${IMAGE}" + echo "EB_APP: ${EB_APP} -- EB_ENV: ${EB_ENV}" + echo "====================================================\n" + } + } + + stage('Build') { + when { + expression { params.DEPLOY_TO == 'dev' } + } + + environment { + ECR_CRED = 'ecr:us-east-1:AWS_CREDENTIALS' + + BUILD_ARGS = + "--build-arg DOMAIN=dev.buildingservice.blocpower.io " + + "--build-arg NUM_PROCESSES=4 " + + "--build-arg NUM_THREADS=4 " + } + + steps { + echo 'Building...' + + sh("""sed -i "s@git+ssh:\\/\\/git@git+https:\\/\\/$GITHUB_OAUTH_TOKEN@" requirements.txt""") + + withCredentials([[ + $class: 'AmazonWebServicesCredentialsBinding', + credentialsId: 'AWS_CREDENTIALS', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' + ]]) { + + sh("eval \$(aws ecr get-login --no-include-email | sed 's|https://||')") + + script { + docker.build(IMAGE, "$BUILD_ARGS .") + docker.withRegistry("https://${env.ECR_REPO_HOST}", ECR_CRED) { + docker.image(IMAGE).push() + } + } + } + + echo 'Clean up images' + sh "docker rmi $IMAGE" + sh(""" docker rmi "${env.ECR_REPO_HOST}/$IMAGE" """) + } + } + + stage('Test') { + steps { + echo "Testing comming soon" + } + } + + stage('Deploy') { + when { + expression { params.DEPLOY_TO != NEVER_DEPLOY_ENV } + } + + steps { + echo "Starting deployment to $EB_ENV" + + echo "Updating EB dockerrun image name to ${env.ECR_REPO_HOST}/$IMAGE" + sh(""" + jq '.Image.Name = "${env.ECR_REPO_HOST}/$IMAGE"' Dockerrun.aws.json | \ + sponge Dockerrun.aws.json + """) + + sh 'zip -r deployment.zip Dockerrun.aws.json .ebextensions/*' + + withCredentials([[ + $class: 'AmazonWebServicesCredentialsBinding', + credentialsId: 'AWS_CREDENTIALS', + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' + ]]) { + sh "eb use $EB_ENV" + sh "eb deploy $EB_ENV" + } + } + } + } + + post { + success { + echo 'Build succeeded' + } + } +} -- GitLab From 0c09eaf594a52e5a36c89a0f288908a3dbae862e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 15:09:07 -0500 Subject: [PATCH 24/37] Discard old builds --- Dockerfile | 2 +- Jenkinsfile | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b400bfd..d02b799 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ ARG NUM_THREADS ENV NUM_THREADS=${NUM_THREADS} # Install dependencies. -RUN apt-get -qq update && apt-get -qq -y install \ +RUN apt-get -qq update && apt-get install -y \ build-essential \ git \ libssl-dev \ diff --git a/Jenkinsfile b/Jenkinsfile index efb23a4..5c72507 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,6 +18,10 @@ pipeline { GITHUB_OAUTH_TOKEN = "${env.GITHUB_CREDENTIALS_PSW}" } + options { + buildDiscarder(logRotator(numToKeepStr: '60')) + } + parameters { choice( name: 'DEPLOY_TO', -- GitLab From 17468244a7c7e07b300bac8d944c043138195340 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 16:01:41 -0500 Subject: [PATCH 25/37] Add slack notification to jenkinsfile --- Jenkinsfile | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5c72507..e48d26b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -156,7 +156,43 @@ pipeline { post { success { - echo 'Build succeeded' + script { + MESSAGE = "$PROJECT test ran successfully (commit hash: $COMMIT_HASH)" + + if (params.DEPLOY_TO != 'none') { + MESSAGE = "$PROJECT built to ${params.DEPLOY_TO} (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" + } + + slackSend( + color: 'good', + message: "$MESSAGE" + ) + + if (params.DEPLOY_TO == 'prod') { + slackSend( + color: 'good', + channel: '#dev', + message: "New version of $PROJECT (${params.VERSION}) in production (commit hash: $COMMIT_HASH)" + ) + } + } + } + + failure { + script { + MESSAGE = "$PROJECT test or build failed (commit hash: $COMMIT_HASH)" + + if (params.DEPLOY_TO == 'dev' || params.DEPLOY_TO == 'staging') { + MESSAGE = "$PROJECT build or test failed [environment: ${params.DEPLOY_TO}] (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" + } else if (params.DEPLOY_TO == 'prod') { + MESSAGE = "$PROJECT failed to build to ${params.DEPLOY_TO} (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" + } + + slackSend( + color: 'danger', + message: $MESSAGE + ) + } } } } -- GitLab From 686f396c07478ca4cbad5ddc3ebeb1826cfdac70 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 16:06:24 -0500 Subject: [PATCH 26/37] Simplify failure step --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e48d26b..e497b77 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -182,9 +182,9 @@ pipeline { script { MESSAGE = "$PROJECT test or build failed (commit hash: $COMMIT_HASH)" - if (params.DEPLOY_TO == 'dev' || params.DEPLOY_TO == 'staging') { - MESSAGE = "$PROJECT build or test failed [environment: ${params.DEPLOY_TO}] (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" - } else if (params.DEPLOY_TO == 'prod') { + if (params.DEPLOY_TO == 'dev') { + MESSAGE = "$PROJECT build or test failed in ${params.DEPLOY_TO} (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" + } else { MESSAGE = "$PROJECT failed to build to ${params.DEPLOY_TO} (version: ${params.VERSION}, commit hash: $COMMIT_HASH)" } -- GitLab From 94e621384c5f4ef17eaebb8a0c9a2d42b3f8af52 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 16:23:08 -0500 Subject: [PATCH 27/37] Add elastic beanstalk config to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1594ce1..eb13367 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__ app/config/*.py !app/config/*.default.py config.json +.elasticbeanstalk/config.yml # PyCharm .idea -- GitLab From 52d0fc360937494e095bfcd0353428fc78bbd7e5 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 17:32:47 -0500 Subject: [PATCH 28/37] Remove condition --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e497b77..4d98df7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { PROJECT = "${EB_APP.replaceAll("\\s", "").toLowerCase()}" EB_ENV = "$PROJECT-${params.DEPLOY_TO}" - if (params.DEPLOY_TO == 'staging' || params.DEPLOY_TO == 'prod' || params.VERSION != 'none') { + if (params.DEPLOY_TO == 'staging' || params.DEPLOY_TO == 'prod') { git( url: "https://${GITHUB_OAUTH_TOKEN}@github.com/Blocp/${PROJECT}.git", branch: "${BRANCH_NAME}" -- GitLab From d5bc73bbb1be9f31eae2d6107fa74f7d868d41bb Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 20:34:42 -0500 Subject: [PATCH 29/37] Add default eb config --- .elasticbeanstalk/config.default.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .elasticbeanstalk/config.default.yml diff --git a/.elasticbeanstalk/config.default.yml b/.elasticbeanstalk/config.default.yml new file mode 100644 index 0000000..fc8a13d --- /dev/null +++ b/.elasticbeanstalk/config.default.yml @@ -0,0 +1,9 @@ +deploy: + artifact: deployment.zip +global: + application_name: iotservice + default_ec2_keyname: EastCoastBPKey + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on + 64bit Amazon Linux/2.8.1 + default_region: us-east-1 + sc: git -- GitLab From 3c98be640fecdb9c2249e8f057beb53bd90b4b77 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 22:56:07 -0500 Subject: [PATCH 30/37] Retag doker image after deploying --- Jenkinsfile | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4d98df7..515a4a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -148,7 +148,31 @@ pipeline { secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' ]]) { sh "eb use $EB_ENV" - sh "eb deploy $EB_ENV" + sh "eb deploy $EB_ENV --label HASH:${COMMIT_HASH}-VERSION:${params.VERSION}" + } + + // Update image with environment deployed to + build( + job: 'Retag project in ECR', + parameters: [ + string(name: 'REPO_NAME', value: "$PROJECT"), + string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), + string(name: 'NEW_IMAGE_TAG', value: "$DEPLOY_TO"), + ] + ) + + script { + if (VERSION != 'none') { + // Update image with version number + build( + job: 'Retag project in ECR', + parameters: [ + string(name: 'REPO_NAME', value: "$PROJECT"), + string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), + string(name: 'NEW_IMAGE_TAG', value: "$VERSION"), + ] + ) + } } } } -- GitLab From ffbed01a4cdc7ac0bb1ba122103b4d1cb0f26b87 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 23:02:05 -0500 Subject: [PATCH 31/37] Add params to new image tag value --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 515a4a6..97146de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -157,7 +157,7 @@ pipeline { parameters: [ string(name: 'REPO_NAME', value: "$PROJECT"), string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), - string(name: 'NEW_IMAGE_TAG', value: "$DEPLOY_TO"), + string(name: 'NEW_IMAGE_TAG', value: "${params.DEPLOY_TO}"), ] ) @@ -169,7 +169,7 @@ pipeline { parameters: [ string(name: 'REPO_NAME', value: "$PROJECT"), string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), - string(name: 'NEW_IMAGE_TAG', value: "$VERSION"), + string(name: 'NEW_IMAGE_TAG', value: "${params.VERSION}"), ] ) } -- GitLab From 0769127b27c0d47bbf8bc3db07484e58b2f944da Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 16 Jan 2018 23:16:31 -0500 Subject: [PATCH 32/37] Update jenkinsfile domain and variable name --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 97146de..fb5c229 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ pipeline { ECR_CRED = 'ecr:us-east-1:AWS_CREDENTIALS' BUILD_ARGS = - "--build-arg DOMAIN=dev.buildingservice.blocpower.io " + + "--build-arg DOMAIN=blocpower.io " + "--build-arg NUM_PROCESSES=4 " + "--build-arg NUM_THREADS=4 " } @@ -162,7 +162,7 @@ pipeline { ) script { - if (VERSION != 'none') { + if (params.VERSION != "none") { // Update image with version number build( job: 'Retag project in ECR', -- GitLab From 0091455d3d28545193d9ad7f9356583d41acabb6 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Wed, 17 Jan 2018 09:30:58 -0500 Subject: [PATCH 33/37] Comment out separate build in jenkinsfile --- Jenkinsfile | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fb5c229..2a85730 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -152,28 +152,28 @@ pipeline { } // Update image with environment deployed to - build( - job: 'Retag project in ECR', - parameters: [ - string(name: 'REPO_NAME', value: "$PROJECT"), - string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), - string(name: 'NEW_IMAGE_TAG', value: "${params.DEPLOY_TO}"), - ] - ) - - script { - if (params.VERSION != "none") { - // Update image with version number - build( - job: 'Retag project in ECR', - parameters: [ - string(name: 'REPO_NAME', value: "$PROJECT"), - string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), - string(name: 'NEW_IMAGE_TAG', value: "${params.VERSION}"), - ] - ) - } - } + // build( + // job: 'Retag project in ECR', + // parameters: [ + // string(name: 'REPO_NAME', value: "$PROJECT"), + // string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), + // string(name: 'NEW_IMAGE_TAG', value: "${params.DEPLOY_TO}"), + // ] + // ) + + // script { + // if (params.VERSION != "none") { + // // Update image with version number + // build( + // job: 'Retag project in ECR', + // parameters: [ + // string(name: 'REPO_NAME', value: "$PROJECT"), + // string(name: 'IMAGE_TAG', value: "$COMMIT_HASH"), + // string(name: 'NEW_IMAGE_TAG', value: "${params.VERSION}"), + // ] + // ) + // } + // } } } } -- GitLab From d87ad4e94734667406c73193729d90e4e6550a44 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Wed, 17 Jan 2018 09:51:15 -0500 Subject: [PATCH 34/37] Wrap failure in quotes --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2a85730..79f8817 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -214,7 +214,7 @@ pipeline { slackSend( color: 'danger', - message: $MESSAGE + message: "$MESSAGE" ) } } -- GitLab From 450e57a78c89bce78a55536882ac6e4ee007a499 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 19 Jan 2018 14:58:15 -0500 Subject: [PATCH 35/37] Add condition for staging --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 79f8817..0a7b1ff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { PROJECT = "${EB_APP.replaceAll("\\s", "").toLowerCase()}" EB_ENV = "$PROJECT-${params.DEPLOY_TO}" - if (params.DEPLOY_TO == 'staging' || params.DEPLOY_TO == 'prod') { + if ((params.DEPLOY_TO == 'staging' && params.VERSION != 'none') || params.DEPLOY_TO == 'prod') { git( url: "https://${GITHUB_OAUTH_TOKEN}@github.com/Blocp/${PROJECT}.git", branch: "${BRANCH_NAME}" -- GitLab From 2417c3ec2a29e89bcda17ac17c56f5dc1ee82be2 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Mon, 22 Jan 2018 15:29:41 -0500 Subject: [PATCH 36/37] Update requirements inside dockerfile and clean up dockerfile --- Dockerfile | 16 +++++++--------- Jenkinsfile | 5 ++--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index d02b799..c9eaf76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,10 @@ FROM ubuntu:18.04 +ARG GITHUB_TOKEN +ARG DOMAIN +ARG NUM_PROCESSES +ARG NUM_THREADS + ENV CODEROOT=/home/docker/code # Log enironment variables. @@ -8,14 +13,6 @@ ENV NGINXREQ=/var/log/app/nginx.req.log ENV UWSGIERR=/var/log/app/uwsgi.err.log ENV UWSGIREQ=/var/log/app/uwsgi.req.log -# Custom environment variables. These change from project to project. -ARG DOMAIN -ENV DOMAIN=${DOMAIN} -ARG NUM_PROCESSES -ENV NUM_PROCESSES=${NUM_PROCESSES} -ARG NUM_THREADS -ENV NUM_THREADS=${NUM_THREADS} - # Install dependencies. RUN apt-get -qq update && apt-get install -y \ build-essential \ @@ -23,7 +20,7 @@ RUN apt-get -qq update && apt-get install -y \ libssl-dev \ libffi-dev \ nginx \ - python-dev \ + python3-dev \ python3 \ python3-pip \ supervisor \ @@ -32,6 +29,7 @@ RUN apt-get -qq update && apt-get install -y \ RUN pip3 install -q uwsgi COPY ./requirements.txt $CODEROOT/requirements.txt +RUN sed -i "s@git+ssh:\/\/git@git+https:\/\/$GITHUB_OAUTH_TOKEN@" $CODEROOT/requirements.txt # Install application requirements. RUN pip3 install -q -r $CODEROOT/requirements.txt diff --git a/Jenkinsfile b/Jenkinsfile index 0a7b1ff..027bba6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -88,14 +88,13 @@ pipeline { BUILD_ARGS = "--build-arg DOMAIN=blocpower.io " + "--build-arg NUM_PROCESSES=4 " + - "--build-arg NUM_THREADS=4 " + "--build-arg NUM_THREADS=4 " + + "--build-arg GITHUB_OAUTH_TOKEN=$GITHUB_OAUTH_TOKEN" } steps { echo 'Building...' - sh("""sed -i "s@git+ssh:\\/\\/git@git+https:\\/\\/$GITHUB_OAUTH_TOKEN@" requirements.txt""") - withCredentials([[ $class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'AWS_CREDENTIALS', -- GitLab From 758fb879de3025af711f97cee93eb80a335a1085 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Mon, 22 Jan 2018 15:46:43 -0500 Subject: [PATCH 37/37] Update arg name --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c9eaf76..6d4f8f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG GITHUB_TOKEN +ARG GITHUB_OAUTH_TOKEN ARG DOMAIN ARG NUM_PROCESSES ARG NUM_THREADS -- GitLab