diff --git a/.ebextensions/log-streaming.config b/.ebextensions/log-streaming.config new file mode 100644 index 0000000000000000000000000000000000000000..6f51a581e1b592917e5f27529ecd0748d625490a --- /dev/null +++ b/.ebextensions/log-streaming.config @@ -0,0 +1,46 @@ +option_settings: + aws:elasticbeanstalk:cloudwatch:logs: + StreamLogs: true + DeleteOnTerminate: false + RetentionInDays: 180 + +packages: + yum: + awslogs: [] + +files: + "/etc/awslogs/config/logs.conf" : + mode: "000600" + owner: root + group: root + content: | + [/var/log/eb-docker/nginx.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.req.log + + [/var/log/eb-docker/nginx.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/nginx.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/nginx.err.log + + [/var/log/eb-docker/uwsgi.req.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.req.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.req.log + + [/var/log/eb-docker/uwsgi.err.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/uwsgi.err.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/uwsgi.err.log + + [/var/log/eb-docker/flask.log] + log_group_name = `{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/log/eb-docker/flask.log"]]}` + log_stream_name = {instance_id} + file = /var/log/eb-docker/containers/eb-current-app/flask.log + +commands: + "01": + command: chkconfig awslogs on + "02": + command: service awslogs restart diff --git a/.ebextensions/managed-platform-update.config b/.ebextensions/managed-platform-update.config new file mode 100644 index 0000000000000000000000000000000000000000..44bdf4a1bf957b9230b46726221c027e9f21bab7 --- /dev/null +++ b/.ebextensions/managed-platform-update.config @@ -0,0 +1,7 @@ +option_settings: + aws:elasticbeanstalk:managedactions: + ManagedActionsEnabled: true + PreferredStartTime: "Mon:07:00" + aws:elasticbeanstalk:managedactions:platformupdate: + UpdateLevel: minor + InstanceRefreshEnabled: false diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml index cd558c9735f25e36cd220c7eea685b88c92b1996..8e85b810c8cac3545fb18f80c00e243605f4df3c 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.yml @@ -4,7 +4,7 @@ branch-defaults: global: application_name: $EB_APP default_ec2_keyname: EastCoastBPKey - default_platform: 64bit Amazon Linux 2015.09 v2.0.8 running Docker 1.9.1 + default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Docker running on 64bit Amazon Linux/2.8.1 default_region: us-east-1 profile: null sc: git diff --git a/Dockerfile b/Dockerfile index 6c06b222d3c3214f81a9c7bd0a48ae610eee5e43..d607b8fe6bbc31ca7c7019d7813e7ebca0c71ede 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM ubuntu +FROM ubuntu:18.04 ENV CODEROOT=/home/docker/code # Log enironment variables. -ENV NGINXERR=/var/log/nginx.err.log -ENV NGINXREQ=/var/log/nginx.req.log -ENV UWSGIERR=/var/log/uwsgi.err.log -ENV UWSGIREQ=/var/log/uwsgi.req.log +ENV NGINXERR=/var/log/app/nginx.err.log +ENV NGINXREQ=/var/log/app/nginx.req.log +ENV UWSGIERR=/var/log/app/uwsgi.err.log +ENV UWSGIREQ=/var/log/app/uwsgi.req.log # Custom environment variables. These change from project to project. ARG DOMAIN @@ -16,20 +16,20 @@ ENV NUM_PROCESSES=${NUM_PROCESSES} ARG NUM_THREADS ENV NUM_THREADS=${NUM_THREADS} -# Add the nginx reposoitory. We need 1.8 in order to support adding CORS headers to error responses. -RUN apt-get update -RUN apt-get install -y --no-install-recommends software-properties-common -RUN add-apt-repository ppa:nginx/stable - # Install dependencies. -RUN apt-get update -RUN apt-get install -y --no-install-recommends python3 python3-software-properties python3-dev python3-setuptools python3-pip -RUN apt-get install -y --no-install-recommends nginx supervisor -RUN apt-get install -y --no-install-recommends build-essential git -RUN apt-get install -y --no-install-recommends libpq-dev -RUN apt-get install -y --no-install-recommends libffi-dev +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + libssl-dev \ + libffi-dev \ + nginx \ + python-dev \ + python3 \ + python3-pip \ + supervisor \ + && rm -rf /var/lib/apt/lists/* + RUN pip3 install uwsgi -RUN rm -rf /var/lib/apt/lists/* COPY ./requirements.txt $CODEROOT/requirements.txt @@ -49,7 +49,7 @@ RUN \ RUN useradd -ms /bin/bash www # Create the log files. -RUN \ +RUN mkdir /var/log/app/ && \ touch $NGINXERR && touch $NGINXREQ && \ touch $UWSGIERR && touch $UWSGIREQ diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json index d334d795f3e285cefe8e903ea2479d5cdad07ab5..e8817f3dabd01777bf3981333327a1312bd34531 100644 --- a/Dockerrun.aws.json +++ b/Dockerrun.aws.json @@ -12,5 +12,6 @@ "HostPort": "80", "ContainerPort": "80" } - ] + ], + "Logging": "/var/log/app" } diff --git a/app/__init__.py b/app/__init__.py index cb425bfb3ace7e6444a6f3a8f651756dfbaf0941..0af6b2d19dcc4e4a2883198cc1257210c42aa466 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -5,7 +5,7 @@ from flask import Flask MEGABYTE = 10**6 LOG_FORMAT = '[%(asctime)s] %(pathname)s:%(lineno)d %(levelname)s - %(message)s' -LOG_PATH = '/var/log/flask.log' +LOG_PATH = '/var/log/app/flask.log' def create_app(config): @@ -15,12 +15,17 @@ def create_app(config): if config != 'config/local.py': handler = RotatingFileHandler(LOG_PATH, maxBytes=MEGABYTE, backupCount=1) - - handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt='%Y-%m-%d %H:%M:%S')) + if not app.debug: + handler.setLevel(logging.INFO) + app.logger.setLevel(logging.INFO) + else: + app.logger.handlers.pop() # remove DebugHandler + app.logger.setLevel(logging.DEBUG) + handler.setLevel(logging.DEBUG) + app.logger.addHandler(handler) - app.logger.setLevel(logging.INFO) app.logger.info('Setting up application...') diff --git a/app/lib/auth0.py b/app/lib/auth0.py index 3014a0a877acf02778d047e4944312048c488128..d6317a27690ac1e281002cca9a5be757ea14c556 100644 --- a/app/lib/auth0.py +++ b/app/lib/auth0.py @@ -11,6 +11,7 @@ class FlaskAuth0: app.config.get('AUTH0_CLIENT_SECRET'), 'https://{}/api/v2/'.format(app.config.get('AUTH0_DOMAIN'))) + auth0_ = FlaskAuth0() diff --git a/app/lib/database.py b/app/lib/database.py index 9bb84f0050838b745d8a1933afdec0c938caebd2..86e3658f5aa0cb239efd29731122ec08ee22b7ec 100644 --- a/app/lib/database.py +++ b/app/lib/database.py @@ -7,11 +7,13 @@ from flask.ext.sqlalchemy import SQLAlchemy # and operations should be through this object. db = SQLAlchemy() + def sqlite_fk_pragma(dbapi_con, con_record): """Apply a one-time pragma to enforce foreign keys in SQLite.""" if isinstance(dbapi_con, sqlite3.Connection): dbapi_con.execute('pragma foreign_keys=ON') + def register(app): """Apply database hacks.""" db.init_app(app) @@ -19,6 +21,7 @@ def register(app): with app.app_context(): event.listen(db.engine, 'connect', sqlite_fk_pragma) + def commit(): """A wrapper for db.session.commit(). @@ -31,6 +34,7 @@ def commit(): db.session.rollback() raise e + class ProcTable: """ProcTable represents a simplified class of SQLAlchemy db.model""" @@ -42,12 +46,14 @@ class ProcTable: """List all columns""" return [column.key for column in self.columns] + class ProcColumn: """ProcColumn represents a column similar to SQLAlchemy column""" def __init__(self, key): self.key = key + def proc(model, method, limit=None, offset=None, **kwargs): """ Run stored procedure diff --git a/app/lib/exceptions.py b/app/lib/exceptions.py index 3a131423eb951a751875570f5f6e443f7fd668e4..b81bf82d84ad7e49bd7453d24dfd0cff1c78b800 100644 --- a/app/lib/exceptions.py +++ b/app/lib/exceptions.py @@ -2,6 +2,7 @@ from werkzeug.exceptions import ( default_exceptions, HTTPException, InternalServerError) from flask import jsonify + def register(app): """Wrap all exceptions so that they render to json.""" def jsonify_error(e): diff --git a/app/models/application.py b/app/models/application.py index da61f03eb96dad8d122d9af46e47bf1c879ab34e..58931f670ca071f902662daf43b0372b75eb7d24 100644 --- a/app/models/application.py +++ b/app/models/application.py @@ -30,10 +30,10 @@ class Application(Model, Tracked, db.Model): # fields, so for now we have to use a list of strings. When 1.1 leaves # beta, this should be ported to the Enum below. # - #class AuthMethods(Enum): - # """An enum for authentication methods.""" - # secret = 'secret' - # referrer = 'referrer' + # class AuthMethods(Enum): + # """An enum for authentication methods.""" + # secret = 'secret' + # referrer = 'referrer' auth_methods = ['secret', 'referrer'] name = db.Column(db.Unicode(length=255), nullable=False) diff --git a/app/models/base.py b/app/models/base.py index bb0f150b22a70973e0557c3a60a3c210d1536613..31acd31a95712b570353486ea9d262232a3190b0 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -1,6 +1,6 @@ -import arrow import decimal import uuid +import arrow from sqlalchemy.sql import func from sqlalchemy.ext.declarative import declared_attr diff --git a/app/permissions/application.py b/app/permissions/application.py index 79d18e42a6d76288f687d6fc4cace5780704cdba..7eb2aec1fc283cd3a3f0591fd6e87cf21d5fad78 100644 --- a/app/permissions/application.py +++ b/app/permissions/application.py @@ -2,8 +2,8 @@ from werkzeug.exceptions import Unauthorized, Forbidden from app.lib.database import db -from .base import Permission from app.models.application import Application +from .base import Permission class ValidApplicationNeed(Permission): @@ -32,12 +32,16 @@ class ValidApplicationNeed(Permission): if not application: self.message = 'No application exists with that key.' + current_app.info(self.message) return False if not application.check_authentication(secret, referrer): self.message = 'Application failed authorization check.' + current_app.info(self.message) return False return True + + app_need = ValidApplicationNeed() diff --git a/app/permissions/auth.py b/app/permissions/auth.py index 2b70882a8c8f509c2ea26a25509ed304ceb893ec..525f5aca583e6fdfde2edde85db345bebe956f43 100644 --- a/app/permissions/auth.py +++ b/app/permissions/auth.py @@ -1,7 +1,7 @@ """Permissions to check authentication.""" import json import requests -from flask import current_app, g +from flask import g from werkzeug.exceptions import Unauthorized from jose import jwt import json @@ -18,8 +18,9 @@ class AuthNeed(Permission): super().__init__() def is_met(self): - from flask import session, request, current_app """Check for authentication with Auth0.""" + + from flask import session, request, current_app auth0_header = current_app.config.get('AUTH0_AUTH_HEADER') auth0_token = request.headers.get(auth0_header) @@ -64,5 +65,6 @@ class AuthNeed(Permission): return True return False + auth_need = AuthNeed() standard_login_need = app_need & auth_need diff --git a/app/scripts/register.py b/app/scripts/register.py index 9c76a7d566ac0a23942b88bc9bde212ac690ca5b..ec58344deb5440a12348b06e9a2fb05d69fedf0a 100644 --- a/app/scripts/register.py +++ b/app/scripts/register.py @@ -151,10 +151,10 @@ class AuthScript(object): Application_ = aliased(Application) if relationship == 'client': q = q.join(Application_, - Application_.id == Authentication.client_id) + Application_.id == Authentication.client_id) elif relationship == 'server': q = q.join(Application_, - Application_.id == Authentication.server_id) + Application_.id == Authentication.server_id) else: raise ValueError( 'Bad relationship: {}.'.format(relationship)) @@ -238,12 +238,12 @@ class RoleScript(object): if relationship == 'client': Application_ = aliased(Application) q = q.join(Application_, - Application_.id == Authentication.client_id) + Application_.id == Authentication.client_id) filters.append(Application_.name == name) elif relationship == 'server': Application_ = aliased(Application) q = q.join(Application_, - Application_.id == Authentication.server_id) + Application_.id == Authentication.server_id) filters.append(Application_.name == name) elif relationship == 'role': filters.append(Role.name == name) @@ -256,13 +256,13 @@ class RoleScript(object): for role in q.all(): server = db.session.query(Application)\ .join(Authentication, - Authentication.server_id == Application.id)\ + Authentication.server_id == Application.id)\ .join(Role, Role.auth_id == Authentication.id)\ .filter(Role.id == role.id)\ .first() client = db.session.query(Application)\ .join(Authentication, - Authentication.client_id == Application.id)\ + Authentication.client_id == Application.id)\ .join(Role, Role.auth_id == Authentication.id)\ .filter(Role.id == role.id)\ .first() diff --git a/app/views/__init__.py b/app/views/__init__.py index 45db929dd4c340e8836dfe6e18de21c710695aee..420107f483873ccf2238e9ccdd51049769026550 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -4,7 +4,7 @@ from app.views import application def register(app): """Register each view to the application. - + This can be used as a comprehensive list of all views that are present in the application. """ diff --git a/app/views/application.py b/app/views/application.py index 54f1adf551bc6e0f2be41015880fab95c7d03865..14552775d58503e338b420f516169ad12418e872 100644 --- a/app/views/application.py +++ b/app/views/application.py @@ -10,7 +10,7 @@ from app.permissions.application import AppNeed, app_need class AuthenticationView(UnprotectedRestView): """A view for app authentications.""" route_base = '/auth/' - decorators = [app_need,] + decorators = [app_need] def get_controller(self): """Return an instance of the authentication controller.""" @@ -46,7 +46,7 @@ class AuthenticationView(UnprotectedRestView): class RoleView(UnprotectedRestView): """A view for auth roles.""" - decorators = [app_need,] + decorators = [app_need] def get_controller(self): """Return an instance of the role controller.""" diff --git a/requirements.txt b/requirements.txt index 0d5349af13620dba2c59fde2f310a453bbedfdf7..82ac6effc40fbb148969f76001aa1e3323c2f10a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,9 @@ arrow==0.7.0 bcrypt==2.0.0 cffi==1.5.2 blessed==1.9.5 +boto3==1.4.4 botocore==1.5.48 -git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.0 +git+ssh://git@github.com/Blocp/bpvalve.git@v1.3.1 cement==2.4.0 colorama==0.3.3 docker-py==1.1.0 @@ -23,16 +24,16 @@ oauth2client==2.0.1 parse==1.6.6 pathspec==0.3.3 postgres==2.2.1 -psycopg2==2.6.1 +psycopg2==2.7.3.2 py==1.4.31 pycparser==2.14 python-dateutil==2.5.3 python-jose==1.3.2 PyYAML==3.11 redis==2.10.5 -requests==2.6.2 +requests==2.7 rsa==3.3 -six==1.10.0 +six==1.11.0 SQLAlchemy==1.0.13 texttable==0.8.4 wcwidth==0.1.6