From b07ee095459f9a6ea2b5b7f21a3ffd373bad9139 Mon Sep 17 00:00:00 2001 From: akkking Date: Tue, 12 Mar 2019 18:21:02 -0400 Subject: [PATCH 1/4] Update nginx config, automation scripts, env files --- .env.default | 3 ++- docker-compose.yml | 19 +++++++++++++++++++ dockerfiles/bloclink.dockerfile | 18 ++++++++++++++++++ entrypoints/bloclink-entry.sh | 4 ++++ nginx/conf.d/services.conf | 15 +++++++++++++++ scripts/populate_env.py | 3 +++ setup.sh | 8 ++++++-- 7 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 dockerfiles/bloclink.dockerfile create mode 100644 entrypoints/bloclink-entry.sh diff --git a/.env.default b/.env.default index cfad506..4e67b72 100644 --- a/.env.default +++ b/.env.default @@ -15,6 +15,7 @@ WEATHER_SERVICE_URI=0.0.0.0:5406 REPORT_SERVICE_URI=0.0.0.0:5407 IOT_SERVICE_URI=0.0.0.0:5408 BLOCNOTE_URI=0.0.0.0:5409 +BLOCLINK_URI=0.0.0.0:5410 # Box.pem Locaton for Document Service BOX_RSA_PRIVATE_CERTIFICATE_PATH=./box.pem @@ -52,8 +53,8 @@ REACT_APP_IOT_SERVICE REACT_APP_USER_SERVICE REACT_APP_WEATHER_SERVICE REACT_APP_BLOCNOTE_URL +REACT_APP_BLOCLINK_URL REACT_APP_COUCH_DB_URL=https://dev.mobile-audit.sg.blocpower.us:4984/audit-dev/ -REACT_APP_BLOCLINK_URL=https://staging.bloclink.blocpower.io # AUTH0 REACT_APP_AUTH0_DOMAIN diff --git a/docker-compose.yml b/docker-compose.yml index 417d8c3..f59c10d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -230,6 +230,25 @@ services: - ../microservices/blocnote/:/app - ../libraries/bpfin/:/bpfin + bloclink: + build: + context: ../ + dockerfile: ./development/dockerfiles/bloclink.dockerfile + args: + service_path: ./microservices/bloclink + environment: + - ENVIRONMENT="local" + env_file: + - ../microservices/bloclink/.env + logging: + driver: "json-file" + options: + max-size: ${LOG_MAX_SIZE} + max-file: ${LOG_MAX_FILE} + ports: ["5309:5309"] + volumes: + - ../microservices/bloclink/:/app + nginx: container_name: nginx command: [nginx-debug, '-g', 'daemon off;'] diff --git a/dockerfiles/bloclink.dockerfile b/dockerfiles/bloclink.dockerfile new file mode 100644 index 0000000..fe515dc --- /dev/null +++ b/dockerfiles/bloclink.dockerfile @@ -0,0 +1,18 @@ +FROM python:3.6.4 + +# set work directory +WORKDIR /app + +ARG service_path + +COPY $service_path/requirements* /app/ + +# install dependencies +RUN pip install -r /app/requirements-dev.txt + +# Start the app +RUN mkdir /entrypoints +COPY ./development/entrypoints/bloclink-entry.sh /entrypoints/ +RUN chmod +x /entrypoints/*.sh + +CMD ["python", "manage.py", "runserver", "0:5310"] diff --git a/entrypoints/bloclink-entry.sh b/entrypoints/bloclink-entry.sh new file mode 100644 index 0000000..ec4334f --- /dev/null +++ b/entrypoints/bloclink-entry.sh @@ -0,0 +1,4 @@ +#! /bin/bash + +# Start Django App +python manage.py runserver 0:5310 diff --git a/nginx/conf.d/services.conf b/nginx/conf.d/services.conf index 2ecc488..da0d5b9 100644 --- a/nginx/conf.d/services.conf +++ b/nginx/conf.d/services.conf @@ -144,3 +144,18 @@ server { # Blocnote. proxy_pass http://blocnote:5309; } } +server { # Bloclink. + large_client_header_buffers 16 32k; + listen 5410; + server_name 0.0.0.0; + + include ./conf.d/error-location.inc; + + location / { + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'HEAD, GET, POST, PUT, DELETE, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'x-blocpower-app-key,x-blocpower-app-secret,x-blocpower-app-token,x-blocpower-auth-key,x-blocpower-auth-token,x-blocpower-google-token,x-blocpower-auth0-token' always; + add_header 'Access-Control-Expose-Headers' 'x-blocpower-app-key,x-blocpower-app-secret,x-blocpower-app-token,x-blocpower-auth-key,x-blocpower-auth-token,x-blocpower-google-token,x-blocpower-auth0-token' always; + proxy_pass http://bloclink:5310; + } +} diff --git a/scripts/populate_env.py b/scripts/populate_env.py index 3be2ba7..23b7fda 100644 --- a/scripts/populate_env.py +++ b/scripts/populate_env.py @@ -27,6 +27,7 @@ source_variables = [ 'WEATHER_SERVICE_URI', 'IOT_SERVICE_URI', 'BLOCNOTE_URI', + 'BLOCLINK_URI', 'DASHBOARD_URI', 'AUTH0_DOMAIN' ] @@ -43,6 +44,7 @@ target_variables = [ 'REACT_APP_WEATHER_SERVICE', 'REACT_APP_IOT_SERVICE', 'REACT_APP_BLOCNOTE_URL', + 'REACT_APP_BLOCLINK_URL', 'REACT_APP_AUTH0_CALLBACK_URL', 'REACT_APP_AUTH0_DOMAIN' ] @@ -59,6 +61,7 @@ rules =[ lambda x: 'http://' + x, lambda x: 'http://' + x, lambda x: 'http://' + x, + lambda x: 'http://' + x, lambda x: 'http://' + x + '/callback', lambda x: x ] diff --git a/setup.sh b/setup.sh index 8426f23..63e2864 100644 --- a/setup.sh +++ b/setup.sh @@ -20,6 +20,7 @@ declare -a app_git_urls=( "git@github.com:Blocp/appservice.git" "git@github.com:Blocp/dashboard.git" "git@github.com:Blocp/blocnote.git" + "git@github.com:Blocp/bloclink.git" ) ## Clone Services @@ -35,11 +36,11 @@ echo -e "Cloning done.\n" echo "Populating config variables ..." for d in */ ; do # BlocNote has config.yml already - if [ $d != "blocnote/" ]; then + if [ $d != "blocnote/" ] && [ $d != "bloclink/" ]; then cp "$d"/.elasticbeanstalk/config.default.yml "$d"/.elasticbeanstalk/config.yml fi - if [ $d != "dashboard/" ] && [ $d != "blocnote/" ]; then + if [ $d != "dashboard/" ] && [ $d != "blocnote/" ] && [ $d != "bloclink/" ]; then cp "$d"/app/config/local.default.py "$d"/app/config/local.py fi @@ -47,6 +48,9 @@ for d in */ ; do # Populate .env for each service env_name=`eb list | grep -i "dev"` + if [ $env_name = "" ]; then + env_name=`eb list | grep -i "stg"` + fi eb printenv $env_name | grep -v -e "Environment Variables" -e "ENVIRONMENT" -e "AUTH0_DOMAIN" -e "REDISURI" -e "BOX_RSA_PRIVATE_CERTIFICATE_PATH" | sed 's/ //g' > ".env" echo "$env_name done." -- GitLab From 42c0e519bed43c750abbd9e346202b313e75c31b Mon Sep 17 00:00:00 2001 From: akkking Date: Wed, 13 Mar 2019 14:52:35 -0400 Subject: [PATCH 2/4] Update dockerfile, docker-compose, scripts --- docker-compose.yml | 2 +- dockerfiles/bloclink.dockerfile | 29 +++++++-- dockerfiles/blocnote.dockerfile | 1 - entrypoints/bloclink-entry.sh | 2 + scripts/populate_env.py | 103 +++++++++++++++++++------------- setup.sh | 2 +- 6 files changed, 91 insertions(+), 48 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f59c10d..d7fd975 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -259,7 +259,7 @@ services: max-size: ${LOG_MAX_SIZE} max-file: ${LOG_MAX_FILE} ports: - - "5400-5409:5400-5409" + - "5400-5410:5400-5410" volumes: - "./nginx/nginx.conf:/etc/nginx/nginx.conf" - "./nginx/conf.d/:/etc/nginx/conf.d/" diff --git a/dockerfiles/bloclink.dockerfile b/dockerfiles/bloclink.dockerfile index fe515dc..03da826 100644 --- a/dockerfiles/bloclink.dockerfile +++ b/dockerfiles/bloclink.dockerfile @@ -1,14 +1,35 @@ -FROM python:3.6.4 +# FROM directive instructing base image to build upon +FROM ubuntu:18.04 # set work directory WORKDIR /app ARG service_path - COPY $service_path/requirements* /app/ -# install dependencies -RUN pip install -r /app/requirements-dev.txt +RUN apt-get -qq update && apt-get install -y \ + build-essential git \ + libpcre3 libpcre3-dev \ + libpq-dev libgeos-dev \ + nginx supervisor wget \ + python3 python3-software-properties python3-dev python3-setuptools python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install uwsgi -I +RUN pip3 install numpy -I +RUN pip3 install scipy -I +RUN pip3 install scikit-learn -I +RUN pip3 install git+https://bb3dd9a7a6eab87372d5377d65d370738cd25196:x-oauth-basic@github.com/blocp/python-pptx + +# Install application requirements +RUN pip3 install -r /app/requirements-dev.txt + +# Create the www user. +RUN useradd -ms /bin/bash www +RUN wget https://github.com/NREL/EnergyPlus/releases/download/v8.6.0/EnergyPlus-8.6.0-198c6a3cff-Linux-x86_64.sh +RUN chmod +x EnergyPlus-8.6.0-198c6a3cff-Linux-x86_64.sh +RUN sh -c 'echo "y\n\n" | ./EnergyPlus-8.6.0-198c6a3cff-Linux-x86_64.sh' +RUN rm EnergyPlus-8.6.0-198c6a3cff-Linux-x86_64.sh # Start the app RUN mkdir /entrypoints diff --git a/dockerfiles/blocnote.dockerfile b/dockerfiles/blocnote.dockerfile index 3f08742..8130a70 100644 --- a/dockerfiles/blocnote.dockerfile +++ b/dockerfiles/blocnote.dockerfile @@ -8,7 +8,6 @@ ENV PYTHONUNBUFFERED 1 WORKDIR /app ARG service_path - COPY $service_path/requirements* /app/ RUN pip install numpy==1.12.0 diff --git a/entrypoints/bloclink-entry.sh b/entrypoints/bloclink-entry.sh index ec4334f..5b74a02 100644 --- a/entrypoints/bloclink-entry.sh +++ b/entrypoints/bloclink-entry.sh @@ -1,4 +1,6 @@ #! /bin/bash +# Collect static files +RUN python manage.py collectstatic --noinput # Start Django App python manage.py runserver 0:5310 diff --git a/scripts/populate_env.py b/scripts/populate_env.py index 23b7fda..6f8d043 100644 --- a/scripts/populate_env.py +++ b/scripts/populate_env.py @@ -66,23 +66,37 @@ rules =[ lambda x: x ] -localhost = "0.0.0.0" -blocnote_host = "blocnote" -building_url = localhost + ":5404" -bloclink_url = "https://staging.bloclink.blocpower.io" - -# List to store lines from .env -lines = [] +building_url = "0.0.0.0:5404" +bloclink_url = "0.0.0.0:5410" env_file = "../development/.env" -# List to store lines from BlocNote requirements.txt -blocnote_requirements_lines = [] -blocnote_requirements_file = "../microservices/blocnote/requirements.txt" - -# List to store lines from BlocNote .env -blocnote_lines = [] -blocnote_env_file = "../microservices/blocnote/.env" +files = [ + { + "name": "blocnote", + "env_file": "../microservices/blocnote/.env", + "requirements_file": "../microservices/blocnote/requirements.txt", + "replace_lines":{ + "ALLOWED_HOSTS=": ",0.0.0.0,blocnote" + }, + "append_lines": [ + "BUILDING_URL=" + building_url, + "BLOCLINK_URL=" + bloclink_url, + "STATIC_URL=/static/" + ] + }, + { + "name": "bloclink", + "env_file": "../microservices/bloclink/.env", + "requirements_file": "../microservices/bloclink/requirements.txt", + "replace_lines": [ + { + "ALLOWED_HOSTS=": ",0.0.0.0,bloclink" + } + ], + }, +] +lines = [] # Read development .env, populate lines list with open(env_file, "r") as f: @@ -116,39 +130,46 @@ with open(env_file, "w") as f: f.write(line + "\n") -# Read BlocNote requirements file, populate list -with open(blocnote_requirements_file, "r") as f: - for line in f: - # Remove ending line break - line = line[:-1] +# Loop thru files variable +for file in files: + lines = [] - if "git+https://" in line: - line = "git+https://" + inputs["GITHUB_TOKEN"] + ":x-oauth-basic@github.com/Blocp/bpfin.git@v0.2.3" + # Populate lines list + with open(file["requirements_file"], "r") as f: + for line in f: + # Remove ending line break + line = line[:-1] - blocnote_requirements_lines.append(line) + # Populate git token to lines list + if "git+https://" in line and ":x-oauth-basic" in line: + segments = line.split(":x-oauth-basic") + line = "git+https://" + inputs["GITHUB_TOKEN"] + ":x-oauth-basic" + segments[1] -# Write BlocNote requirements file using list -with open(blocnote_requirements_file, "w") as f: - for line in blocnote_requirements_lines: - f.write(line + "\n") + lines.append(line) + # Write back requirements file with lines list + with open(file["requirements_file"], "w") as f: + for line in lines: + f.write(line + "\n") -# Read BlocNote .env file, populate list -with open(blocnote_env_file, "r") as f: - for line in f: - # Remove ending line break - line = line[:-1] + lines = [] - if "ALLOWED_HOSTS=" in line: - line += "," + localhost + "," + blocnote_host + # Read .env file, populate list + with open(file["env_file"], "r") as f: + for line in f: + # Remove ending line break + line = line[:-1] - blocnote_lines.append(line) + for k, v in file["replace_lines"]: + if k in line: + line += v - blocnote_lines.append("BUILDING_URL=" + building_url) - blocnote_lines.append("BLOCLINK_URL=" + bloclink_url) - blocnote_lines.append("STATIC_URL=/static/") + lines.append(line) -# Write BlocNote .env file using list -with open(blocnote_env_file, "w") as f: - for line in blocnote_lines: - f.write(line + "\n") + for line in file["append_lines"]: + lines.append(line) + + # Write back .env file with lines list + with open(file["env_file"], "w") as f: + for line in lines: + f.write(line + "\n") diff --git a/setup.sh b/setup.sh index 63e2864..73671d1 100644 --- a/setup.sh +++ b/setup.sh @@ -48,7 +48,7 @@ for d in */ ; do # Populate .env for each service env_name=`eb list | grep -i "dev"` - if [ $env_name = "" ]; then + if [ "$env_name" = "" ]; then env_name=`eb list | grep -i "stg"` fi eb printenv $env_name | grep -v -e "Environment Variables" -e "ENVIRONMENT" -e "AUTH0_DOMAIN" -e "REDISURI" -e "BOX_RSA_PRIVATE_CERTIFICATE_PATH" | sed 's/ //g' > ".env" -- GitLab From f5ca209cfb9908f9d1299f7f848032e6f8d205fb Mon Sep 17 00:00:00 2001 From: akkking Date: Wed, 13 Mar 2019 17:04:24 -0400 Subject: [PATCH 3/4] Update script to fix config error --- docker-compose.yml | 2 +- scripts/populate_env.py | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d7fd975..ae333d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -245,7 +245,7 @@ services: options: max-size: ${LOG_MAX_SIZE} max-file: ${LOG_MAX_FILE} - ports: ["5309:5309"] + ports: ["5310:5310"] volumes: - ../microservices/bloclink/:/app diff --git a/scripts/populate_env.py b/scripts/populate_env.py index 6f8d043..945a924 100644 --- a/scripts/populate_env.py +++ b/scripts/populate_env.py @@ -69,7 +69,6 @@ rules =[ building_url = "0.0.0.0:5404" bloclink_url = "0.0.0.0:5410" env_file = "../development/.env" - files = [ { "name": "blocnote", @@ -88,14 +87,12 @@ files = [ "name": "bloclink", "env_file": "../microservices/bloclink/.env", "requirements_file": "../microservices/bloclink/requirements.txt", - "replace_lines": [ - { - "ALLOWED_HOSTS=": ",0.0.0.0,bloclink" - } - ], + "replace_lines":{ + "ALLOWED_HOSTS=": ",0.0.0.0,bloclink" + }, + "append_lines": [] }, ] - lines = [] # Read development .env, populate lines list @@ -160,7 +157,7 @@ for file in files: # Remove ending line break line = line[:-1] - for k, v in file["replace_lines"]: + for k, v in file["replace_lines"].items(): if k in line: line += v -- GitLab From bfa7a804c07c0b961fb560fc96f38214fc50b065 Mon Sep 17 00:00:00 2001 From: akkking Date: Wed, 13 Mar 2019 17:59:32 -0400 Subject: [PATCH 4/4] Add worker and pentaho services --- docker-compose.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ae333d8..a5b5314 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -277,5 +277,40 @@ services: image: redis:alpine ports: ["6380:6379"] + rabbitmq: + image: rabbitmq:3-management + hostname: my-rabbit + environment: + RABBITMQ_DEFAULT_USER: "rabbitmq" + RABBITMQ_DEFAULT_PASS: "rabbitmq" + RABBITMQ_DEFAULT_VHOST: "/" + ports: + - "15672:15672" + - "5672:5672" + + worker: + build: + context: . + dockerfile: ./development/dockerfiles/bloclink/worker.Dockerfile + command: celery -A bloclink worker -l info --logfile /var/log/simulation.log + depends_on: + - rabbitmq + volumes: + - .:/code + - eplus:/eplus_data + + pentaho: + build: + context: . + dockerfile: ./development/dockerfiles/bloclink/pentaho.Dockerfile + command: carte.sh /pentaho-di/carte_config.xml + ports: + - "9081:9081" + volumes: + - pentaho_di:/jobs + - eplus:/eplus_data + volumes: pgdata: + eplus: + pentaho_di: -- GitLab