diff --git a/.ebextensions/log-streaming.config b/.ebextensions/log-streaming.config new file mode 100644 index 0000000000000000000000000000000000000000..6f51a581e1b592917e5f27529ecd0748d625490a --- /dev/null +++ b/.ebextensions/log-streaming.config @@ -0,0 +1,46 @@ +option_settings: + aws:elasticbeanstalk:cloudwatch:logs: + StreamLogs: true + DeleteOnTerminate: false + RetentionInDays: 180 + +packages: + yum: + awslogs: [] + +files: + "/etc/awslogs/config/logs.conf" : + mode: "000600" + owner: root + group: root + content: | + [/var/log/eb-docker/nginx.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.req.log + + [/var/log/eb-docker/nginx.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.err.log + + [/var/log/eb-docker/uwsgi.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.req.log + + [/var/log/eb-docker/uwsgi.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.err.log + + [/var/log/eb-docker/flask.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/flask.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/flask.log + +commands: + "01": + command: chkconfig awslogs on + "02": + command: service awslogs restart diff --git a/.ebextensions/managed-platform-update.config b/.ebextensions/managed-platform-update.config new file mode 100644 index 0000000000000000000000000000000000000000..44bdf4a1bf957b9230b46726221c027e9f21bab7 --- /dev/null +++ b/.ebextensions/managed-platform-update.config @@ -0,0 +1,7 @@ +option_settings: + aws:elasticbeanstalk:managedactions: + ManagedActionsEnabled: true + PreferredStartTime: "Mon:07:00" + aws:elasticbeanstalk:managedactions:platformupdate: + UpdateLevel: minor + InstanceRefreshEnabled: false diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml index cd558c9735f25e36cd220c7eea685b88c92b1996..8e85b810c8cac3545fb18f80c00e243605f4df3c 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.yml @@ -4,7 +4,7 @@ branch-defaults: global: application_name: $EB_APP default_ec2_keyname: EastCoastBPKey - default_platform: 64bit Amazon Linux 2015.09 v2.0.8 running Docker 1.9.1 + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on 64bit Amazon Linux/2.8.1 default_region: us-east-1 profile: null sc: git diff --git a/Dockerfile b/Dockerfile index 4b0148166924c2325629c04f262531855d3d1332..d607b8fe6bbc31ca7c7019d7813e7ebca0c71ede 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM ubuntu:16.10 +FROM ubuntu:18.04 ENV CODEROOT=/home/docker/code # Log enironment variables. -ENV NGINXERR=/var/log/nginx.err.log -ENV NGINXREQ=/var/log/nginx.req.log -ENV UWSGIERR=/var/log/uwsgi.err.log -ENV UWSGIREQ=/var/log/uwsgi.req.log +ENV NGINXERR=/var/log/app/nginx.err.log +ENV NGINXREQ=/var/log/app/nginx.req.log +ENV UWSGIERR=/var/log/app/uwsgi.err.log +ENV UWSGIREQ=/var/log/app/uwsgi.req.log # Custom environment variables. These change from project to project. ARG DOMAIN @@ -16,19 +16,20 @@ ENV NUM_PROCESSES=${NUM_PROCESSES} ARG NUM_THREADS ENV NUM_THREADS=${NUM_THREADS} -# Add the nginx reposoitory. We need 1.8 in order to support adding CORS headers to error responses. -RUN apt-get update -RUN apt-get install -y --no-install-recommends software-properties-common -RUN add-apt-repository ppa:nginx/stable - # Install dependencies. -RUN apt-get update -RUN apt-get install -y --no-install-recommends python3 python3-software-properties python3-dev python3-setuptools python3-pip -RUN apt-get install -y --no-install-recommends nginx supervisor -RUN apt-get install -y --no-install-recommends build-essential git -RUN apt-get install -y --no-install-recommends libpq-dev +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + libssl-dev \ + libffi-dev \ + nginx \ + python-dev \ + python3 \ + python3-pip \ + supervisor \ + && rm -rf /var/lib/apt/lists/* + RUN pip3 install uwsgi -RUN rm -rf /var/lib/apt/lists/* COPY ./requirements.txt $CODEROOT/requirements.txt @@ -48,7 +49,7 @@ RUN \ RUN useradd -ms /bin/bash www # Create the log files. -RUN \ +RUN mkdir /var/log/app/ && \ touch $NGINXERR && touch $NGINXREQ && \ touch $UWSGIERR && touch $UWSGIREQ diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json index d334d795f3e285cefe8e903ea2479d5cdad07ab5..e8817f3dabd01777bf3981333327a1312bd34531 100644 --- a/Dockerrun.aws.json +++ b/Dockerrun.aws.json @@ -12,5 +12,6 @@ "HostPort": "80", "ContainerPort": "80" } - ] + ], + "Logging": "/var/log/app" } diff --git a/app/__init__.py b/app/__init__.py index dc103fa2f517a1a3731ecb86483afcd5e3ad0cbd..71715dc11cf501b4cdfd070e1af3bde0761f0768 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -5,7 +5,7 @@ from flask import Flask MEGABYTE = 10**6 LOG_FORMAT = '[%(asctime)s] %(pathname)s:%(lineno)d %(levelname)s - %(message)s' -LOG_PATH = '/var/log/flask.log' +LOG_PATH = '/var/log/app/flask.log' def create_app(config): @@ -15,12 +15,17 @@ def create_app(config): if config != 'config/local.py': handler = RotatingFileHandler(LOG_PATH, maxBytes=MEGABYTE, backupCount=1) - - handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt='%Y-%m-%d %H:%M:%S')) + if not app.debug: + handler.setLevel(logging.INFO) + app.logger.setLevel(logging.INFO) + else: + app.logger.handlers.pop() # remove DebugHandler + app.logger.setLevel(logging.DEBUG) + handler.setLevel(logging.DEBUG) + app.logger.addHandler(handler) - app.logger.setLevel(logging.INFO) app.logger.info('Setting up application...') diff --git a/app/config/development.default.py b/app/config/development.default.py index 035f9f2518c9187c345f550463f978ba7dcff97f..973810e858ac16a1e7bb9fbd4f46dff824a52e2d 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -14,15 +14,16 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { - 'app': 'http://dev.appservice.blocpower.io', - 'user': 'http://dev.userservice.blocpower.io', + 'app': 'https://dev.appservice.blocpower.io', + 'user': 'https://dev.userservice.blocpower.io', } } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/local.default.py b/app/config/local.default.py index 278bbcb8563875257275b81d2a9b2c94bb472fde..6b8f4da0286987ef566e9fb5ef638fa7c1e85fbc 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -14,7 +14,8 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { 'app': 'http://127.0.0.1:5400', 'user': 'http://127.0.0.1:5401', @@ -22,7 +23,7 @@ SERVICE_CONFIG = { } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/production.default.py b/app/config/production.default.py index 7faf8d862b449452f3f1a0d3c248515611ca6ced..8b383145988d2a5033cb5a6633ee15c3367d870d 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -14,15 +14,16 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { - 'app': 'http://app.s.blocpower.us', - 'user': 'http://user.s.blocpower.us', + 'app': 'https://appservice.blocpower.io', + 'user': 'https://userservice.blocpower.io', } } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/staging.default.py b/app/config/staging.default.py index 2328bd38edf9b92aa90100d5117fab218e8e4b43..ca95f5fe32a250420a80fd183cde75921064f069 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -14,15 +14,16 @@ SERVICE_CONFIG = { 'headers': { 'app_key': 'x-blocpower-app-key', 'app_token': 'x-blocpower-app-token', - 'app_secret': 'x-blocpower-app-secret'}, + 'app_secret': 'x-blocpower-app-secret' + }, 'urls': { - 'app': 'http://staging.app.s.blocpower.us', - 'user': 'http://staging.user.s.blocpower.us' + 'app': 'https://staging.appservice.blocpower.io', + 'user': 'https://staging.userservice.blocpower.io' } } # AppService -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/config/test.default.py b/app/config/test.default.py index 2a2795c4dcab65b27a3f7b22f02b32038df7c742..f879a212b316a271a4c3f9ce602b6df6a362c44f 100644 --- a/app/config/test.default.py +++ b/app/config/test.default.py @@ -24,7 +24,7 @@ SERVICE_CONFIG = { } # App -APP_CACHE_EXPIRY = 60 * 60 # One hour. +APP_CACHE_EXPIRY = 60 * 60 # One hour. # Auth0 Authentication AUTH0_AUTH_HEADER = 'x-blocpower-auth0-token' diff --git a/app/lib/auth0.py b/app/lib/auth0.py index 3014a0a877acf02778d047e4944312048c488128..d6317a27690ac1e281002cca9a5be757ea14c556 100644 --- a/app/lib/auth0.py +++ b/app/lib/auth0.py @@ -11,6 +11,7 @@ class FlaskAuth0: app.config.get('AUTH0_CLIENT_SECRET'), 'https://{}/api/v2/'.format(app.config.get('AUTH0_DOMAIN'))) + auth0_ = FlaskAuth0() diff --git a/app/lib/database.py b/app/lib/database.py index b21112b5f3c08d8897a9077fc462578669831bbc..86e3658f5aa0cb239efd29731122ec08ee22b7ec 100644 --- a/app/lib/database.py +++ b/app/lib/database.py @@ -8,8 +8,10 @@ from flask.ext.sqlalchemy import SQLAlchemy db = SQLAlchemy() -def my_on_connect(dbapi_con, con_record): - print("New DBAPI connection: ", dbapi_con) +def sqlite_fk_pragma(dbapi_con, con_record): + """Apply a one-time pragma to enforce foreign keys in SQLite.""" + if isinstance(dbapi_con, sqlite3.Connection): + dbapi_con.execute('pragma foreign_keys=ON') def register(app): @@ -17,7 +19,7 @@ def register(app): db.init_app(app) with app.app_context(): - event.listen(db.engine, 'connect', my_on_connect) + event.listen(db.engine, 'connect', sqlite_fk_pragma) def commit(): @@ -32,6 +34,7 @@ def commit(): db.session.rollback() raise e + class ProcTable: """ProcTable represents a simplified class of SQLAlchemy db.model""" @@ -43,12 +46,14 @@ class ProcTable: """List all columns""" return [column.key for column in self.columns] + class ProcColumn: """ProcColumn represents a column similar to SQLAlchemy column""" def __init__(self, key): self.key = key + def proc(model, method, limit=None, offset=None, **kwargs): """ Run stored procedure diff --git a/app/lib/exceptions.py b/app/lib/exceptions.py index 3a131423eb951a751875570f5f6e443f7fd668e4..b81bf82d84ad7e49bd7453d24dfd0cff1c78b800 100644 --- a/app/lib/exceptions.py +++ b/app/lib/exceptions.py @@ -2,6 +2,7 @@ from werkzeug.exceptions import ( default_exceptions, HTTPException, InternalServerError) from flask import jsonify + def register(app): """Wrap all exceptions so that they render to json.""" def jsonify_error(e): diff --git a/app/lib/service.py b/app/lib/service.py index d479e36fb0d46af06cbeec938c919c924e412f14..bbfef367e6531f0ca002b9b64c7bda739858b60e 100644 --- a/app/lib/service.py +++ b/app/lib/service.py @@ -28,9 +28,9 @@ class Service(object): """Get a list of headers from the configuration and cache.""" return { self.config['headers']['app_key']: self.config['app_key'], - self.config['headers']['app_token']: \ - self.cache.get(self.cache_key) or '', - self.config['headers']['app_secret']: self.config['app_secret']} + self.config['headers']['app_token']: self.cache.get(self.cache_key) or '', + self.config['headers']['app_secret']: self.config['app_secret'] + } def dispatch(self, method, url, *args, **kargs): """Issue a request.""" diff --git a/app/lib/session.py b/app/lib/session.py index 7fb691c3a5c7e40002d7c2d741812c42e619a8f8..a26cebe8a439df417a241b2ea5ae15d933bfd786 100644 --- a/app/lib/session.py +++ b/app/lib/session.py @@ -19,26 +19,18 @@ class NoCookieSessionInterface(SessionInterface): """Create a session from some headers. Ignore the cookie.""" from app.lib.service import services return NoCookieSession( - user_key=\ - request.headers.get(app.config.get('HEADER_AUTH_KEY')), - user_token=\ - request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), - app_key=\ - request.headers.get(services.config['headers']['app_key']), - app_token=\ - request.headers.get(services.config['headers']['app_token'])) + user_key=request.headers.get(app.config.get('HEADER_AUTH_KEY')), + user_token=request.headers.get(app.config.get('HEADER_AUTH_TOKEN')), + app_key=request.headers.get(services.config['headers']['app_key']), + app_token=request.headers.get(services.config['headers']['app_token'])) def save_session(self, app, session, response): """Respond with some headers from the session. Don't set the cookie.""" from app.lib.service import services - response.headers.set( - app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) - response.headers.set( - app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) - response.headers.set( - services.config['headers']['app_key'], session.get('app_key')) - response.headers.set( - services.config['headers']['app_token'], session.get('app_token')) + response.headers.set(app.config.get('HEADER_AUTH_KEY'), session.get('user_key')) + response.headers.set(app.config.get('HEADER_AUTH_TOKEN'), session.get('user_token')) + response.headers.set(services.config['headers']['app_key'], session.get('app_key')) + response.headers.set(services.config['headers']['app_token'], session.get('app_token')) def register(app): diff --git a/app/models/base.py b/app/models/base.py index ef7e4d215514efb9efa17f63b5f9647a1745e036..6def76fdb3ab2df6911c6ec04297fa1787c1e876 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -1,6 +1,6 @@ -import arrow import decimal import uuid +import arrow from werkzeug.exceptions import InternalServerError, BadRequest import base64 diff --git a/app/permissions/application.py b/app/permissions/application.py index b43f140bfc47fe1fe9b9cb6a049ea63205974fcb..42cc990960d12074b23866d4cb14763a389c9ee1 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -18,6 +18,7 @@ class AppNeed(Permission): # Check for a key. key = session.get('app_key') if not key: + current_app.logger.info('Session app key is empty') return False token = session.get('app_token') @@ -36,27 +37,33 @@ class AppNeed(Permission): # such, we can't use the normal service library for communication. secret = request.headers.get( services.config['headers']['app_secret'], None) - response = requests.get( - services.app.url + '/auth/', - params={ - 'client_key': key, - 'server_key': services.config['app_key']}, - headers={ - services.config['headers']['app_key']: key, - services.config['headers']['app_secret']: secret, - 'referer': request.headers.get('referer')}) + + try: + response = requests.get( + services.app.url + '/auth/', + params={ + 'client_key': key, + 'server_key': services.config['app_key'] + }, + headers={ + services.config['headers']['app_key']: key, + services.config['headers']['app_secret']: secret, + 'referer': request.headers.get('referer') + }) + response.raise_for_status() + except requests.exceptions.RequestException as e: + current_app.logger.error('Failed to communicate with appservice %s', e) + raise e # Check if the app authentication failed. if response.status_code != 200: + current_app.logger.info('App authentication failed') return False data = response.json() - if ( - 'data' not in data or - not isinstance(data['data'], list) or - len(data['data']) == 0 or - 'token' not in data['data'][0] - ): + if ('data' not in data or not isinstance(data['data'], list) + or len(data['data']) == 0 or 'token' not in data['data'][0]): + current_app.logger.error('Token not found in appservice response') return False token = data['data'][0]['token'] @@ -71,8 +78,8 @@ class AppNeed(Permission): # where there is already a token and it expires before we retrieve it. _, token = redis.client_apps.pipeline()\ .set(cache_key, token, - ex=current_app.config.get('APP_CACHE_EXPIRY'), - nx=True)\ + ex=current_app.config.get('APP_CACHE_EXPIRY'), + nx=True)\ .get(cache_key)\ .execute() @@ -80,16 +87,17 @@ class AppNeed(Permission): session['app_token'] = token return True + + app_need = AppNeed() + class RoleNeed(Permission): """Checks if a role is met by hitting the app service.""" @property def error(self): """Return an error based on the role.""" - return Unauthorized( - 'Please authenticate with an application with the {} role.'\ - .format(self.role)) + return Unauthorized(f'Please authenticate with an application with the {self.role} role.') def __init__(self, role): """Initialize with a role.""" diff --git a/app/permissions/auth.py b/app/permissions/auth.py index b7be02c75c6bf4f061af038295c4095f8e2a41bc..299015d2e80c71fdf32dff0f53d8ec1bcefad52a 100644 --- a/app/permissions/auth.py +++ b/app/permissions/auth.py @@ -1,7 +1,7 @@ """Permissions to check authentication.""" import json import requests -from flask import current_app, g +from flask import g from werkzeug.exceptions import Unauthorized from jose import jwt from .base import Permission @@ -15,8 +15,9 @@ class AuthNeed(Permission): super().__init__() def is_met(self): - from flask import session, request, current_app """Check for authentication with Auth0.""" + + from flask import session, request, current_app auth0_header = current_app.config.get('AUTH0_AUTH_HEADER') auth0_token = request.headers.get(auth0_header) @@ -61,5 +62,6 @@ class AuthNeed(Permission): return True return False + auth_need = AuthNeed() standard_login_need = app_need & auth_need diff --git a/requirements.txt b/requirements.txt index 12d0485fe78e033c03713536dc19779a95fd7158..04da7346d4b3ec895d2f81fae9af3a5039e04def 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,22 +25,22 @@ oauth2client==2.0.1 parse==1.6.6 pathspec==0.3.3 postgres==2.2.1 -psycopg2==2.6.1 +psycopg2==2.7.3.2 py==1.4.31 pyasn1==0.1.9 pyasn1-modules==0.0.8 pycparser==2.14 -python-dateutil==2.4.2 +python-dateutil==2.5.3 python-jose==1.3.2 PyYAML==3.11 redis==2.10.5 requests==2.7 rsa==3.3 simplejson==3.8.2 -six==1.10.0 +six==1.11.0 SQLAlchemy==1.0.13 texttable==0.8.4 wcwidth==0.1.6 websocket-client==0.35.0 -Werkzeug==0.11.4 +Werkzeug==0.11.5 WTForms==2.1