From 0936a39778d466633598db7394f7bf05a8f8f396 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:04:43 -0500 Subject: [PATCH 01/20] Update to ubuntu 18 and python 3.6 --- Dockerfile | 21 +++++++++------------ requirements.txt | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5611b79..fab0f4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu +FROM ubuntu:18.04 ENV CODEROOT=/home/docker/code @@ -16,19 +16,16 @@ ENV NUM_PROCESSES=${NUM_PROCESSES} ARG NUM_THREADS ENV NUM_THREADS=${NUM_THREADS} -# Add the nginx reposoitory. We need 1.8 in order to support adding CORS headers to error responses. -RUN apt-get update -RUN apt-get install -y --no-install-recommends software-properties-common -RUN add-apt-repository ppa:nginx/stable - # Install dependencies. -RUN apt-get update -RUN apt-get install -y --no-install-recommends python3 python3-software-properties python3-dev python3-setuptools python3-pip -RUN apt-get install -y --no-install-recommends nginx supervisor -RUN apt-get install -y --no-install-recommends build-essential git -RUN apt-get install -y --no-install-recommends libpq-dev +RUN apt-get update && apt-get install -y \ + git \ + nginx \ + python3 \ + python3-pip \ + supervisor \ + && rm -rf /var/lib/apt/lists/* + RUN pip3 install uwsgi -RUN rm -rf /var/lib/apt/lists/* COPY ./requirements.txt $CODEROOT/requirements.txt diff --git a/requirements.txt b/requirements.txt index 7f1e1df..175ee5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ oauth2client==2.0.1 parse==1.6.6 pathspec==0.3.3 postgres==2.2.1 -psycopg2==2.6.1 +psycopg2==2.7.3.2 py==1.4.31 pyasn1==0.1.9 pyasn1-modules==0.0.8 -- GitLab From a70c625e9644f11724fe96a2270cd5171dc97925 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:23:03 -0500 Subject: [PATCH 02/20] Add python extras to dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index fab0f4d..1a65914 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,12 @@ ENV NUM_THREADS=${NUM_THREADS} # Install dependencies. RUN apt-get update && apt-get install -y \ + build-essential \ git \ + libssl-dev \ + libffi-dev \ nginx \ + python-dev \ python3 \ python3-pip \ supervisor \ -- GitLab From ecf43ac0d190107c118cd2773feead165f86b121 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:23:18 -0500 Subject: [PATCH 03/20] Update requirements to mactch other services --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 175ee5e..c76c3ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ arrow==0.7.0 blessed==1.9.5 +boto3==1.4.4 botocore==1.5.48 git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.0 cement==2.4.0 @@ -27,16 +28,16 @@ py==1.4.31 pyasn1==0.1.9 pyasn1-modules==0.0.8 pycparser==2.14 -python-dateutil==2.4.2 +python-dateutil==2.6.2 python-jose==1.3.2 PyYAML==3.11 redis==2.10.5 -requests==2.6.2 +requests==2.7 rsa==3.3 six==1.10.0 SQLAlchemy==1.0.13 texttable==0.8.4 wcwidth==0.1.6 websocket-client==0.35.0 -Werkzeug==0.11.4 +Werkzeug==0.11.5 WTForms==2.1 -- GitLab From d18c5621e0bcdac646799f909093a0035f7cc794 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:29:44 -0500 Subject: [PATCH 04/20] Fix python dateutil version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c76c3ab..a915976 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ py==1.4.31 pyasn1==0.1.9 pyasn1-modules==0.0.8 pycparser==2.14 -python-dateutil==2.6.2 +python-dateutil==2.5.3 python-jose==1.3.2 PyYAML==3.11 redis==2.10.5 -- GitLab From bcd0a7f175c2fe1ef4539c6752775f978e807300 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 13:32:24 -0500 Subject: [PATCH 05/20] Update six to 1.11.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a915976..cd5ea57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ PyYAML==3.11 redis==2.10.5 requests==2.7 rsa==3.3 -six==1.10.0 +six==1.11.0 SQLAlchemy==1.0.13 texttable==0.8.4 wcwidth==0.1.6 -- GitLab From cd6b7258c399d8f6ea255f62dfc68d0c05903e96 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 2 Jan 2018 19:09:53 -0500 Subject: [PATCH 06/20] Add elasticbeanstalk logging --- Dockerfile | 10 +++++----- Dockerrun.aws.json | 3 ++- app/__init__.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a65914..d607b8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ FROM ubuntu:18.04 ENV CODEROOT=/home/docker/code # Log enironment variables. -ENV NGINXERR=/var/log/nginx.err.log -ENV NGINXREQ=/var/log/nginx.req.log -ENV UWSGIERR=/var/log/uwsgi.err.log -ENV UWSGIREQ=/var/log/uwsgi.req.log +ENV NGINXERR=/var/log/app/nginx.err.log +ENV NGINXREQ=/var/log/app/nginx.req.log +ENV UWSGIERR=/var/log/app/uwsgi.err.log +ENV UWSGIREQ=/var/log/app/uwsgi.req.log # Custom environment variables. These change from project to project. ARG DOMAIN @@ -49,7 +49,7 @@ RUN \ RUN useradd -ms /bin/bash www # Create the log files. -RUN \ +RUN mkdir /var/log/app/ && \ touch $NGINXERR && touch $NGINXREQ && \ touch $UWSGIERR && touch $UWSGIREQ diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json index d334d79..e8817f3 100644 --- a/Dockerrun.aws.json +++ b/Dockerrun.aws.json @@ -12,5 +12,6 @@ "HostPort": "80", "ContainerPort": "80" } - ] + ], + "Logging": "/var/log/app" } diff --git a/app/__init__.py b/app/__init__.py index dc103fa..3fae06a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -5,7 +5,7 @@ from flask import Flask MEGABYTE = 10**6 LOG_FORMAT = '[%(asctime)s] %(pathname)s:%(lineno)d %(levelname)s - %(message)s' -LOG_PATH = '/var/log/flask.log' +LOG_PATH = '/var/log/app/flask.log' def create_app(config): -- GitLab From 31efec99bcd11123009cad64fd5850be7b9f3e3e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Thu, 4 Jan 2018 12:48:05 -0500 Subject: [PATCH 07/20] Update default platform --- .elasticbeanstalk/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml index cd558c9..8e85b81 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.yml @@ -4,7 +4,7 @@ branch-defaults: global: application_name: $EB_APP default_ec2_keyname: EastCoastBPKey - default_platform: 64bit Amazon Linux 2015.09 v2.0.8 running Docker 1.9.1 + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on 64bit Amazon Linux/2.8.1 default_region: us-east-1 profile: null sc: git -- GitLab From 3f40a2e9548fb8d29ca501e448cd94b4b3c4932e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 01:22:34 -0500 Subject: [PATCH 08/20] Remove DebugHandler in development --- app/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 3fae06a..71715dc 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -15,12 +15,17 @@ def create_app(config): if config != 'config/local.py': handler = RotatingFileHandler(LOG_PATH, maxBytes=MEGABYTE, backupCount=1) - - handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt='%Y-%m-%d %H:%M:%S')) + if not app.debug: + handler.setLevel(logging.INFO) + app.logger.setLevel(logging.INFO) + else: + app.logger.handlers.pop() # remove DebugHandler + app.logger.setLevel(logging.DEBUG) + handler.setLevel(logging.DEBUG) + app.logger.addHandler(handler) - app.logger.setLevel(logging.INFO) app.logger.info('Setting up application...') -- GitLab From c7b61ddd33101903eacdb2a9d44edbc4dd4fbf2e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 01:31:27 -0500 Subject: [PATCH 09/20] Add cloudwatch log streams --- .ebextensions/log-streaming.config | 46 ++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .ebextensions/log-streaming.config diff --git a/.ebextensions/log-streaming.config b/.ebextensions/log-streaming.config new file mode 100644 index 0000000..6f51a58 --- /dev/null +++ b/.ebextensions/log-streaming.config @@ -0,0 +1,46 @@ +option_settings: + aws:elasticbeanstalk:cloudwatch:logs: + StreamLogs: true + DeleteOnTerminate: false + RetentionInDays: 180 + +packages: + yum: + awslogs: [] + +files: + "/etc/awslogs/config/logs.conf" : + mode: "000600" + owner: root + group: root + content: | + [/var/log/eb-docker/nginx.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.req.log + + [/var/log/eb-docker/nginx.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.err.log + + [/var/log/eb-docker/uwsgi.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.req.log + + [/var/log/eb-docker/uwsgi.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.err.log + + [/var/log/eb-docker/flask.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/flask.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/flask.log + +commands: + "01": + command: chkconfig awslogs on + "02": + command: service awslogs restart -- GitLab From 13cc7f20633cfc71f9e52544ad328296d7531f4f Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 01:32:03 -0500 Subject: [PATCH 10/20] Add elasticbeanstalk auto platform updater --- .ebextensions/managed-platform-update.config | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .ebextensions/managed-platform-update.config diff --git a/.ebextensions/managed-platform-update.config b/.ebextensions/managed-platform-update.config new file mode 100644 index 0000000..44bdf4a --- /dev/null +++ b/.ebextensions/managed-platform-update.config @@ -0,0 +1,7 @@ +option_settings: + aws:elasticbeanstalk:managedactions: + ManagedActionsEnabled: true + PreferredStartTime: "Mon:07:00" + aws:elasticbeanstalk:managedactions:platformupdate: + UpdateLevel: minor + InstanceRefreshEnabled: false -- GitLab From ed5f941d2a0264ab871e1038066f0d80eff3939e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:00:15 -0500 Subject: [PATCH 11/20] Add logging statements to app need --- app/permissions/application.py | 48 ++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/app/permissions/application.py b/app/permissions/application.py index b43f140..1775397 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -18,6 +18,7 @@ class AppNeed(Permission): # Check for a key. key = session.get('app_key') if not key: + current_app.logger.error('Session app key is empty') return False token = session.get('app_token') @@ -36,27 +37,33 @@ class AppNeed(Permission): # such, we can't use the normal service library for communication. secret = request.headers.get( services.config['headers']['app_secret'], None) - response = requests.get( - services.app.url + '/auth/', - params={ - 'client_key': key, - 'server_key': services.config['app_key']}, - headers={ - services.config['headers']['app_key']: key, - services.config['headers']['app_secret']: secret, - 'referer': request.headers.get('referer')}) + + try: + response = requests.get( + services.app.url + '/auth/', + params={ + 'client_key': key, + 'server_key': services.config['app_key'] + }, + headers={ + services.config['headers']['app_key']: key, + services.config['headers']['app_secret']: secret, + 'referer': request.headers.get('referer') + }) + response.raise_for_status() + except requests.exceptions.RequestException as e: + current_app.logger.error('Failed to communicate with appservice %s', e) + raise e # Check if the app authentication failed. if response.status_code != 200: + current_app.logger.error('App authentication failed') return False data = response.json() - if ( - 'data' not in data or - not isinstance(data['data'], list) or - len(data['data']) == 0 or - 'token' not in data['data'][0] - ): + if ('data' not in data or not isinstance(data['data'], list) + or len(data['data']) == 0 or 'token' not in data['data'][0]): + current_app.logger.error('Token not found in appservice response') return False token = data['data'][0]['token'] @@ -71,8 +78,8 @@ class AppNeed(Permission): # where there is already a token and it expires before we retrieve it. _, token = redis.client_apps.pipeline()\ .set(cache_key, token, - ex=current_app.config.get('APP_CACHE_EXPIRY'), - nx=True)\ + ex=current_app.config.get('APP_CACHE_EXPIRY'), + nx=True)\ .get(cache_key)\ .execute() @@ -80,16 +87,17 @@ class AppNeed(Permission): session['app_token'] = token return True + + app_need = AppNeed() + class RoleNeed(Permission): """Checks if a role is met by hitting the app service.""" @property def error(self): """Return an error based on the role.""" - return Unauthorized( - 'Please authenticate with an application with the {} role.'\ - .format(self.role)) + return Unauthorized(f'Please authenticate with an application with the {self.role} role.') def __init__(self, role): """Initialize with a role.""" -- GitLab From 3bbe7606f74fc63394843ec5bd086f0e31994063 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:01:17 -0500 Subject: [PATCH 12/20] Fix pylint and pep8 errors --- app/config/development.default.py | 2 +- app/config/local.default.py | 2 +- app/config/production.default.py | 2 +- app/config/staging.default.py | 2 +- app/config/test.default.py | 2 +- app/lib/auth0.py | 1 + app/lib/database.py | 6 ++++++ app/lib/exceptions.py | 1 + app/lib/service.py | 6 +++--- app/lib/session.py | 24 ++++++++---------------- app/models/base.py | 2 +- app/permissions/auth.py | 6 ++++-- app/views/__init__.py | 2 +- 13 files changed, 30 insertions(+), 28 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 216af3c..9b13b8b 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -22,7 +22,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/local.default.py b/app/config/local.default.py index a53dfc0..fef073e 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -19,7 +19,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/production.default.py b/app/config/production.default.py index d03bb1f..1d8a34d 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -22,7 +22,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/staging.default.py b/app/config/staging.default.py index 21bdbcb..d6b0bed 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -22,7 +22,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/test.default.py b/app/config/test.default.py index 6639a96..a187093 100644 --- a/app/config/test.default.py +++ b/app/config/test.default.py @@ -19,7 +19,7 @@ SERVICE_CONFIG = { } # App -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Test applications TEST_CLIENTS = { diff --git a/app/lib/auth0.py b/app/lib/auth0.py index 3014a0a..d6317a2 100644 --- a/app/lib/auth0.py +++ b/app/lib/auth0.py @@ -11,6 +11,7 @@ class FlaskAuth0: app.config.get('AUTH0_CLIENT_SECRET'), 'https://{}/api/v2/'.format(app.config.get('AUTH0_DOMAIN'))) + auth0_ = FlaskAuth0() diff --git a/app/lib/database.py b/app/lib/database.py index 9bb84f0..86e3658 100644 --- a/app/lib/database.py +++ b/app/lib/database.py @@ -7,11 +7,13 @@ from flask.ext.sqlalchemy import SQLAlchemy # and operations should be through this object. db = SQLAlchemy() + def sqlite_fk_pragma(dbapi_con, con_record): """Apply a one-time pragma to enforce foreign keys in SQLite.""" if isinstance(dbapi_con, sqlite3.Connection): dbapi_con.execute('pragma foreign_keys=ON') + def register(app): """Apply database hacks.""" db.init_app(app) @@ -19,6 +21,7 @@ def register(app): with app.app_context(): event.listen(db.engine, 'connect', sqlite_fk_pragma) + def commit(): """A wrapper for db.session.commit(). @@ -31,6 +34,7 @@ def commit(): db.session.rollback() raise e + class ProcTable: """ProcTable represents a simplified class of SQLAlchemy db.model""" @@ -42,12 +46,14 @@ class ProcTable: """List all columns""" return [column.key for column in self.columns] + class ProcColumn: """ProcColumn represents a column similar to SQLAlchemy column""" def __init__(self, key): self.key = key + def proc(model, method, limit=None, offset=None, **kwargs): """ Run stored procedure diff --git a/app/lib/exceptions.py b/app/lib/exceptions.py index 3a13142..b81bf82 100644 --- a/app/lib/exceptions.py +++ b/app/lib/exceptions.py @@ -2,6 +2,7 @@ from werkzeug.exceptions import ( default_exceptions, HTTPException, InternalServerError) from flask import jsonify + def register(app): """Wrap all exceptions so that they render to json.""" def jsonify_error(e): diff --git a/app/lib/service.py b/app/lib/service.py index d479e36..bbfef36 100644 --- a/app/lib/service.py +++ b/app/lib/service.py @@ -28,9 +28,9 @@ class Service(object): """Get a list of headers from the configuration and cache.""" return { self.config['headers']['app_key']: self.config['app_key'], - self.config['headers']['app_token']: \ - self.cache.get(self.cache_key) or '', - self.config['headers']['app_secret']: self.config['app_secret']} + self.config['headers']['app_token']: self.cache.get(self.cache_key) or '', + self.config['headers']['app_secret']: self.config['app_secret'] + } def dispatch(self, method, url, *args, **kargs): """Issue a request.""" diff --git a/app/lib/session.py b/app/lib/session.py index 7fb691c..a26cebe 100644 --- a/app/lib/session.py +++ b/app/lib/session.py @@ -19,26 +19,18 @@ class NoCookieSessionInterface(SessionInterface): """Create a session from some headers. Ignore the cookie.""" from app.lib.service import services return NoCookieSession( - user_key=\ - request.headers.get(app.config.get('HEADER_AUTH_KEY')), - user_token=\ - request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), - app_key=\ - request.headers.get(services.config['headers']['app_key']), - app_token=\ - request.headers.get(services.config['headers']['app_token'])) + user_key=request.headers.get(app.config.get('HEADER_AUTH_KEY')), + user_token=request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), + app_key=request.headers.get(services.config['headers']['app_key']), + app_token=request.headers.get(services.config['headers']['app_token'])) def save_session(self, app, session, response): """Respond with some headers from the session. Don't set the cookie.""" from app.lib.service import services - response.headers.set( - app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) - response.headers.set( - app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) - response.headers.set( - services.config['headers']['app_key'], session.get('app_key')) - response.headers.set( - services.config['headers']['app_token'], session.get('app_token')) + response.headers.set(app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) + response.headers.set(app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) + response.headers.set(services.config['headers']['app_key'], session.get('app_key')) + response.headers.set(services.config['headers']['app_token'], session.get('app_token')) def register(app): diff --git a/app/models/base.py b/app/models/base.py index bb0f150..31acd31 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -1,6 +1,6 @@ -import arrow import decimal import uuid +import arrow from sqlalchemy.sql import func from sqlalchemy.ext.declarative import declared_attr diff --git a/app/permissions/auth.py b/app/permissions/auth.py index b7be02c..299015d 100644 --- a/app/permissions/auth.py +++ b/app/permissions/auth.py @@ -1,7 +1,7 @@ """Permissions to check authentication.""" import json import requests -from flask import current_app, g +from flask import g from werkzeug.exceptions import Unauthorized from jose import jwt from .base import Permission @@ -15,8 +15,9 @@ class AuthNeed(Permission): super().__init__() def is_met(self): - from flask import session, request, current_app """Check for authentication with Auth0.""" + + from flask import session, request, current_app auth0_header = current_app.config.get('AUTH0_AUTH_HEADER') auth0_token = request.headers.get(auth0_header) @@ -61,5 +62,6 @@ class AuthNeed(Permission): return True return False + auth_need = AuthNeed() standard_login_need = app_need & auth_need diff --git a/app/views/__init__.py b/app/views/__init__.py index 1dcf967..015e00a 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -3,7 +3,7 @@ def register(app): """Register each view to the application. - + This can be used as a comprehensive list of all views that are present in the application. """ -- GitLab From c06076d42506bd219f6c6f030c83f1cdff7b9728 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:19:05 -0500 Subject: [PATCH 13/20] Update service urls to https --- app/config/development.default.py | 3 ++- app/config/local.default.py | 3 ++- app/config/production.default.py | 7 ++++--- app/config/staging.default.py | 7 ++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 9b13b8b..02edda6 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -14,7 +14,8 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { 'app': 'http://dev.appservice.blocpower.io', 'user': 'http://dev.userservice.blocpower.io', diff --git a/app/config/local.default.py b/app/config/local.default.py index fef073e..605fa43 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -11,7 +11,8 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { 'app': 'http://127.0.0.1:5400', 'user': 'http://127.0.0.1:5401', diff --git a/app/config/production.default.py b/app/config/production.default.py index 1d8a34d..2c17d09 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -14,10 +14,11 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { - 'app': 'http://app.s.blocpower.us', - 'user': 'http://user.s.blocpower.us', + 'app': 'https://appservice.blocpower.io', + 'user': 'https://userservice.blocpower.io', } } diff --git a/app/config/staging.default.py b/app/config/staging.default.py index d6b0bed..bc3fccc 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -14,10 +14,11 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { - 'app': 'http://staging.app.s.blocpower.us', - 'user': 'http://staging.user.s.blocpower.us' + 'app': 'https://staging.appservice.blocpower.io', + 'user': 'https://staging.userservice.blocpower.io' } } -- GitLab From b61918a0e38e83abade9c16fbdbc44ea115f24d6 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:39:35 -0500 Subject: [PATCH 14/20] Bump bpvalve to v1.3.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd5ea57..35b57ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ arrow==0.7.0 blessed==1.9.5 boto3==1.4.4 botocore==1.5.48 -git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.0 +git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.1 cement==2.4.0 colorama==0.3.3 docker-py==1.1.0 -- GitLab From 481670041996f51073ce0cfc9efaca46007ea895 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:50:04 -0500 Subject: [PATCH 15/20] Remove service and session files --- app/lib/service.py | 93 ---------------------------------------------- app/lib/session.py | 38 ------------------- 2 files changed, 131 deletions(-) delete mode 100644 app/lib/service.py delete mode 100644 app/lib/session.py diff --git a/app/lib/service.py b/app/lib/service.py deleted file mode 100644 index bbfef36..0000000 --- a/app/lib/service.py +++ /dev/null @@ -1,93 +0,0 @@ -"""A library for inter-service communication.""" -import requests - - -class Service(object): - """A wrapper for requests to services.""" - # A default implementation of the cache. This is just a dictionary. - cache = {} - - def __init__(self, key, config, cache=None, cache_prefix=''): - """Sets information from config.""" - self.key = key - self.cache = cache or self.cache - self.cache_prefix = cache_prefix - self.config = config - try: - self.url = config['urls'][key] - except KeyError: - raise ValueError('No service found with key {}.'.format(key)) - - @property - def cache_key(self): - """The key to use when storing data in the cache.""" - return self.cache_prefix + self.key - - @property - def headers(self): - """Get a list of headers from the configuration and cache.""" - return { - self.config['headers']['app_key']: self.config['app_key'], - self.config['headers']['app_token']: self.cache.get(self.cache_key) or '', - self.config['headers']['app_secret']: self.config['app_secret'] - } - - def dispatch(self, method, url, *args, **kargs): - """Issue a request.""" - headers = {} - headers.update(self.headers) - headers.update(kargs.get('headers', {})) - kargs.update({'headers': headers}) - response = getattr(requests, method)(self.url + url, *args, **kargs) - self.cache.set( - self.cache_key, - response.headers.get(self.config['headers']['app_token'])) - return response - - def get(self, *args, **kargs): - """Issue a get request.""" - return self.dispatch('get', *args, **kargs) - - def post(self, url, *args, **kargs): - """Issue a post request.""" - return self.dispatch('post', *args, **kargs) - - def put(self, url, *args, **kargs): - """Issue a put request.""" - return self.dispatch('put', *args, **kargs) - - def delete(self, url, *args, **kargs): - """Issue a delete request.""" - return self.dispatch('delete', *args, **kargs) - - -class ServiceInterface(object): - """A wrapper for retrieving services from the current app.""" - def init_app(self, app): - """Set the application.""" - self.flask_app = app - - @property - def config(self): - """The configuration.""" - return self.flask_app.config.get('SERVICE_CONFIG') - - def __getattr__(self, key): - """Get the requested service object from app config.""" - from app.lib.red import redis - try: - return Service( - key, - self.config, - redis.server_apps, - self.config['app_key'] + '/') - except ValueError as e: - raise AttributeError(e) - - -services = ServiceInterface() - - -def register(app): - """Set the application on the interface.""" - services.init_app(app) diff --git a/app/lib/session.py b/app/lib/session.py deleted file mode 100644 index a26cebe..0000000 --- a/app/lib/session.py +++ /dev/null @@ -1,38 +0,0 @@ -"""An in-memory session. - - This is associated with each request and provides a convenient way to pass - around data that would normally be associated with a cookie. Since our - application is stateless, this is not communicated with the frontend. There - is no actual cookie at use here. -""" -from flask.sessions import SessionMixin, SessionInterface - - -class NoCookieSession(dict, SessionMixin): - """A session to pass around.""" - pass - - -class NoCookieSessionInterface(SessionInterface): - """An interface to the no cookie session.""" - def open_session(self, app, request): - """Create a session from some headers. Ignore the cookie.""" - from app.lib.service import services - return NoCookieSession( - user_key=request.headers.get(app.config.get('HEADER_AUTH_KEY')), - user_token=request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), - app_key=request.headers.get(services.config['headers']['app_key']), - app_token=request.headers.get(services.config['headers']['app_token'])) - - def save_session(self, app, session, response): - """Respond with some headers from the session. Don't set the cookie.""" - from app.lib.service import services - response.headers.set(app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) - response.headers.set(app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) - response.headers.set(services.config['headers']['app_key'], session.get('app_key')) - response.headers.set(services.config['headers']['app_token'], session.get('app_token')) - - -def register(app): - """Set the session interface on the app.""" - app.session_interface = NoCookieSessionInterface() -- GitLab From ada73dad257ff815e8a3b7a329a1e1dc04e43b97 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 15:56:18 -0500 Subject: [PATCH 16/20] Fix pep8 style --- app/models/application.py | 8 ++++---- app/permissions/application.py | 2 ++ app/scripts/register.py | 12 ++++++------ app/views/application.py | 4 ++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/app/models/application.py b/app/models/application.py index da61f03..58931f6 100644 --- a/app/models/application.py +++ b/app/models/application.py @@ -30,10 +30,10 @@ class Application(Model, Tracked, db.Model): # fields, so for now we have to use a list of strings. When 1.1 leaves # beta, this should be ported to the Enum below. # - #class AuthMethods(Enum): - # """An enum for authentication methods.""" - # secret = 'secret' - # referrer = 'referrer' + # class AuthMethods(Enum): + # """An enum for authentication methods.""" + # secret = 'secret' + # referrer = 'referrer' auth_methods = ['secret', 'referrer'] name = db.Column(db.Unicode(length=255), nullable=False) diff --git a/app/permissions/application.py b/app/permissions/application.py index 79d18e4..97fe165 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -38,6 +38,8 @@ class ValidApplicationNeed(Permission): return False return True + + app_need = ValidApplicationNeed() diff --git a/app/scripts/register.py b/app/scripts/register.py index 9c76a7d..ec58344 100644 --- a/app/scripts/register.py +++ b/app/scripts/register.py @@ -151,10 +151,10 @@ class AuthScript(object): Application_ = aliased(Application) if relationship == 'client': q = q.join(Application_, - Application_.id == Authentication.client_id) + Application_.id == Authentication.client_id) elif relationship == 'server': q = q.join(Application_, - Application_.id == Authentication.server_id) + Application_.id == Authentication.server_id) else: raise ValueError( 'Bad relationship: {}.'.format(relationship)) @@ -238,12 +238,12 @@ class RoleScript(object): if relationship == 'client': Application_ = aliased(Application) q = q.join(Application_, - Application_.id == Authentication.client_id) + Application_.id == Authentication.client_id) filters.append(Application_.name == name) elif relationship == 'server': Application_ = aliased(Application) q = q.join(Application_, - Application_.id == Authentication.server_id) + Application_.id == Authentication.server_id) filters.append(Application_.name == name) elif relationship == 'role': filters.append(Role.name == name) @@ -256,13 +256,13 @@ class RoleScript(object): for role in q.all(): server = db.session.query(Application)\ .join(Authentication, - Authentication.server_id == Application.id)\ + Authentication.server_id == Application.id)\ .join(Role, Role.auth_id == Authentication.id)\ .filter(Role.id == role.id)\ .first() client = db.session.query(Application)\ .join(Authentication, - Authentication.client_id == Application.id)\ + Authentication.client_id == Application.id)\ .join(Role, Role.auth_id == Authentication.id)\ .filter(Role.id == role.id)\ .first() diff --git a/app/views/application.py b/app/views/application.py index 54f1adf..1455277 100644 --- a/app/views/application.py +++ b/app/views/application.py @@ -10,7 +10,7 @@ from app.permissions.application import AppNeed, app_need class AuthenticationView(UnprotectedRestView): """A view for app authentications.""" route_base = '/auth/' - decorators = [app_need,] + decorators = [app_need] def get_controller(self): """Return an instance of the authentication controller.""" @@ -46,7 +46,7 @@ class AuthenticationView(UnprotectedRestView): class RoleView(UnprotectedRestView): """A view for auth roles.""" - decorators = [app_need,] + decorators = [app_need] def get_controller(self): """Return an instance of the role controller.""" -- GitLab From 45f2af3e66fd69d53b7a07bf0ae247136e94ce28 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 16:00:10 -0500 Subject: [PATCH 17/20] Switch log leve in AppNedd --- app/permissions/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/permissions/application.py b/app/permissions/application.py index 1775397..42cc990 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -18,7 +18,7 @@ class AppNeed(Permission): # Check for a key. key = session.get('app_key') if not key: - current_app.logger.error('Session app key is empty') + current_app.logger.info('Session app key is empty') return False token = session.get('app_token') @@ -57,7 +57,7 @@ class AppNeed(Permission): # Check if the app authentication failed. if response.status_code != 200: - current_app.logger.error('App authentication failed') + current_app.logger.info('App authentication failed') return False data = response.json() -- GitLab From 4d6c7daf2c5d5960a976caaa6890a77bb78354e8 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 16:03:50 -0500 Subject: [PATCH 18/20] Add logging statements to ValidApplicationNeed --- app/permissions/application.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/permissions/application.py b/app/permissions/application.py index 97fe165..c8e06f4 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -32,9 +32,11 @@ class ValidApplicationNeed(Permission): if not application: self.message = 'No application exists with that key.' + current_app.info(self.message) return False if not application.check_authentication(secret, referrer): self.message = 'Application failed authorization check.' + current_app.info(self.message) return False return True -- GitLab From 439b9f763bfe4ed7a86badb1e204d61e37fdba05 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 16:05:36 -0500 Subject: [PATCH 19/20] Sort imports --- app/permissions/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/permissions/application.py b/app/permissions/application.py index c8e06f4..7eb2aec 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -2,8 +2,8 @@ from werkzeug.exceptions import Unauthorized, Forbidden from app.lib.database import db -from .base import Permission from app.models.application import Application +from .base import Permission class ValidApplicationNeed(Permission): -- GitLab From 5c8d9a9072abb4c6fad2294adbcfb3bd2fe1d816 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Fri, 5 Jan 2018 16:14:53 -0500 Subject: [PATCH 20/20] Add https to dev config --- app/config/development.default.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 02edda6..d94b0b9 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -17,8 +17,8 @@ SERVICE_CONFIG = { 'app_secret': 'x-blocpower-app-secret' }, 'urls': { - 'app': 'http://dev.appservice.blocpower.io', - 'user': 'http://dev.userservice.blocpower.io', + 'app': 'https://dev.appservice.blocpower.io', + 'user': 'https://dev.userservice.blocpower.io', } } -- GitLab