diff --git a/Jenkinsfile b/Jenkinsfile index 094cb8061408097a478f54c18a82c68989ec397d..1cf519522fa7b7808823fd6232a33226315b959a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -158,7 +158,7 @@ def deploy(String envname) { usernameVariable: 'JENKINS_ERL_CI_USER')]) { sh 'cd infra/ansible && ansible-playbook \ - -i inventory/' + envname + '.yml \ + -i inventory/' + envname + '.ini \ -e project_package=${DEPLOY_PACKAGE} \ -e project_release=${DEPLOY_RELEASE} \ -u ${JENKINS_ERL_CI_USER} \ diff --git a/infra/ansible/ansible.cfg b/infra/ansible/ansible.cfg index a6ab924c20f29ded4a97cc58f1fb5000d1ef1818..279cc76ae39f3bbe2d903da8790735912f80efff 100644 --- a/infra/ansible/ansible.cfg +++ b/infra/ansible/ansible.cfg @@ -1,6 +1,7 @@ [defaults] host_key_checking = False timeout = 30 +interpreter_python = auto_silent [ssh_connection] ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=1800s diff --git a/infra/ansible/deploy.yml b/infra/ansible/deploy.yml index 8c76ed74a1211da9b94a5673e441d6ecf8ca3b7b..319ac32370acbd241ed659fb35f45bb9fb8457f8 100644 --- a/infra/ansible/deploy.yml +++ b/infra/ansible/deploy.yml @@ -3,7 +3,9 @@ hosts: appservers vars: - project_root: "/home/nynja/erlang-server" + project_home: "/home/nynja" + project_root: "{{ project_home}}/erlang-server" + project_backups_dir: "{{ backups_dir | default('/home/nynja/backups') }}" project_shared_path: "{{ project_root }}/shared" project_source_path: "{{ project_root }}/shared/source" project_database_name: "Mnesia.mq2@{{ ansible_fqdn }}" @@ -23,6 +25,11 @@ - path: log src: log project_api_port: 8888 + project_mqtt_ports: [8083,8084,8883,1883] + # Can be set in the inventory variables or CLI + project_drop_mqtt_traffic: "{{ drop_mqtt_traffic | default(true) }}" + project_auto_backups: "{{ auto_backups | default(false) }}" + project_deploy_backups: "{{ deploy_backups | default(false) }}" tasks: ### Initialize @@ -30,24 +37,24 @@ deploy_helper: "path={{ project_root }} release={{ project_release|default(omit) }} state=present shared_path={{ project_shared_path }}" ### Update source + - name: Ensure root source dir is present + file: "path={{ project_source_path }} state=directory" + - name: Download remote package get_url: url: "{{ project_package }}" dest: "{{ project_source_path }}/{{ project_package|basename }}" checksum: "{{ project_package_checksum|default(omit) }}" when: "'http' in project_package" - # register: remote_package - name: Upload local package copy: src: "{{ project_package }}" dest: "{{ project_source_path }}/{{ project_package|basename }}" when: "'http' not in project_package" - # register: local_package - name: Ensure sources dir is present file: "path={{ project_source_path }}/{{ deploy_helper.new_release }} state=directory" - # when: remote_package.changed or local_package.changed register: new_src - name: Extract the package @@ -91,6 +98,27 @@ file: "path='{{ deploy_helper.new_release_path }}/{{ item.path }}' src='{{ deploy_helper.shared_path }}/{{ item.src }}' state=link" with_items: "{{ project_shared_children }}" + ### Setup backups + - name: Make sure ~/bin exists + file: + path: "{{ project_home }}/bin" + state: directory + mode: '0755' + + - name: Install backup script + copy: + src: files/backup.sh + dest: "{{ project_home }}/bin/backup.sh" + mode: '0700' + + - name: Setup backups cronjob + cron: + name: "backup erlang server database" + minute: "0" + hour: "*/6" + job: "{{ project_home }}/bin/backup.sh {{ deploy_helper.current_path }} {{ project_backups_dir }}" + state: "{{ 'present' if project_auto_backups else 'absent' }}" + ### Finalize - name: Check server binary stat: @@ -103,6 +131,38 @@ register: ping when: binary.stat.exists == True + # Create a backup + - name: Create pre-deploy backup + command: + cmd: "{{ project_home }}/bin/backup.sh {{ deploy_helper.current_path }} {{ project_backups_dir }}" + when: + - binary.stat.exists == True + - ping.stdout == "pong" + - project_deploy_backups|bool + + - name: Dump Mnesia log + command: + cmd: "{{ deploy_helper.current_path }}/bin/server eval 'mnesia:dump_log()'" + when: binary.stat.exists == True and ping.stdout == "pong" + + - name: Drop MQTT traffic + iptables: + chain: INPUT + action: insert + protocol: tcp + destination_port: "{{ item | int }}" + jump: DROP + state: present + with_items: "{{ project_mqtt_ports }}" + become: yes + become_user: root + when: project_drop_mqtt_traffic | bool + + - name: Clean MQTT subscription tables + command: + cmd: "{{ deploy_helper.current_path }}/bin/server eval '[ mnesia:clear_table(X) || X <- [mqtt_subscription, mqtt_subscriber, mqtt_subproperty]]'" + when: binary.stat.exists == True and ping.stdout == "pong" + - name: Stop erlang server command: "{{ deploy_helper.current_path }}/bin/server stop" when: binary.stat.exists == True and ping.stdout == "pong" @@ -129,3 +189,16 @@ timeout: 10 status_code: 200 tags: [health-check] + + - name: Allow MQTT traffic + iptables: + chain: INPUT + action: insert + protocol: tcp + destination_port: "{{ item | int }}" + jump: DROP + state: absent + with_items: "{{ project_mqtt_ports }}" + become: yes + become_user: root + when: project_drop_mqtt_traffic | bool diff --git a/infra/ansible/files/backup.sh b/infra/ansible/files/backup.sh new file mode 100755 index 0000000000000000000000000000000000000000..e4c20b3ae036dfcc8001c9e99b779bb9bf9207ad --- /dev/null +++ b/infra/ansible/files/backup.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e + +BACKUP_TIME=`date '+%a_%d_%b_%Y_%H_%M_%S'` +DATE=`date '+%Y-%m-%dT%H'` +SERVER_DIR=${1:?"Server directory must be provided as first argument"} +BACKUP_DIR=${2:?"Backup directory must be provided as second argument"} +BACKUP_DBNAME="mq2" +BACKUP_HOST=`hostname -A | xargs` +BACKUP_FILE_REL="${BACKUP_DBNAME}@${BACKUP_HOST}_bkp_${DATE}*" +BACKUP_FILE="${SERVER_DIR}/backup/${BACKUP_FILE_REL}" +BACKUP_ARCHIVE=${BACKUP_DBNAME}@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz +BACKUP_LOG="${BACKUP_DIR}/log/backup_nodetool.log" + + +mkdir -p ${BACKUP_DIR} +mkdir -p ${BACKUP_DIR}/log + +echo "Deleting old backupfile. Rention Time is 30 days." >> ${BACKUP_LOG} +find ${BACKUP_DIR} -name "mq*" -type f -mtime +30 -print -delete >> ${BACKUP_LOG} + + +echo "Backup Time: ${BACKUP_TIME}" >> ${BACKUP_LOG} + +${SERVER_DIR}/bin/server eval 'roster_db:backup()' + +echo "Backup File:" +ls -ltr ${BACKUP_FILE} + +echo "Moving backup file to ${BACKUP_DIR}/" >> ${BACKUP_LOG} +mv ${BACKUP_FILE} ${BACKUP_DIR}/ >> ${BACKUP_LOG} + +echo "Compressing backup file" >> ${BACKUP_LOG} +cd ${BACKUP_DIR} +tar -cJf ${BACKUP_ARCHIVE} ${BACKUP_FILE_REL} >> ${BACKUP_LOG} +chmod 640 ${BACKUP_ARCHIVE} +rm -rf ${BACKUP_DIR}/${BACKUP_FILE_REL} >> ${BACKUP_LOG} + +echo "Compressed Backup File:" >> ${BACKUP_LOG} +ls -ltr ${BACKUP_DIR}/${BACKUP_ARCHIVE} >> ${BACKUP_LOG} + +echo "Log File:" >> ${BACKUP_LOG} +ls -ltr ${BACKUP_LOG} >> ${BACKUP_LOG} + +echo "Job Completed" >> ${BACKUP_LOG} +echo "--------------------------------" >> ${BACKUP_LOG} diff --git a/infra/ansible/inventory/dev1.yml b/infra/ansible/inventory/dev1.ini similarity index 100% rename from infra/ansible/inventory/dev1.yml rename to infra/ansible/inventory/dev1.ini diff --git a/infra/ansible/inventory/dev2.yml b/infra/ansible/inventory/dev2.ini similarity index 100% rename from infra/ansible/inventory/dev2.yml rename to infra/ansible/inventory/dev2.ini diff --git a/infra/ansible/inventory/prod.ini b/infra/ansible/inventory/prod.ini new file mode 100644 index 0000000000000000000000000000000000000000..538adc38ef795f4128a7d4f3cf2f87a8f80dc69c --- /dev/null +++ b/infra/ansible/inventory/prod.ini @@ -0,0 +1,8 @@ +[appservers] +35.230.49.68 + +[appservers:vars] +drop_mqtt_traffic=true +auto_backups=true +deploy_backups=true +backups_dir=/backup/erlang-server diff --git a/infra/ansible/inventory/prod.yml b/infra/ansible/inventory/prod.yml deleted file mode 100644 index 4403aa4630b96f7dd5ca670b30f313fc2315a583..0000000000000000000000000000000000000000 --- a/infra/ansible/inventory/prod.yml +++ /dev/null @@ -1,2 +0,0 @@ -[appservers] -35.230.49.68 diff --git a/infra/ansible/inventory/stage1.ini b/infra/ansible/inventory/stage1.ini new file mode 100644 index 0000000000000000000000000000000000000000..a716bf19b71b95e481e86f55442962387dacdd2f --- /dev/null +++ b/infra/ansible/inventory/stage1.ini @@ -0,0 +1,8 @@ +[appservers] +35.197.66.129 + +[appservers:vars] +drop_mqtt_traffic=true +auto_backups=true +deploy_backups=true +backups_dir=/backup/erlang-server diff --git a/infra/ansible/inventory/stage1.yml b/infra/ansible/inventory/stage1.yml deleted file mode 100644 index 18a1228ef76e97f6ea0fdb12e8a45a57bfad3df1..0000000000000000000000000000000000000000 --- a/infra/ansible/inventory/stage1.yml +++ /dev/null @@ -1,2 +0,0 @@ -[appservers] -35.197.66.129 diff --git a/infra/ansible/inventory/stage2.yml b/infra/ansible/inventory/stage2.ini similarity index 100% rename from infra/ansible/inventory/stage2.yml rename to infra/ansible/inventory/stage2.ini diff --git a/infra/ansible/setup.yml b/infra/ansible/setup.yml new file mode 100644 index 0000000000000000000000000000000000000000..b50b138569469ebfa57b2a0fe0decac72434f01a --- /dev/null +++ b/infra/ansible/setup.yml @@ -0,0 +1,13 @@ +--- +- name: Setup app servers + hosts: appservers + + tasks: + - name: Allow 'nynja' user to run iptables + lineinfile: + dest: /etc/sudoers + state: present + regexp: '^nynja' + line: 'nynja ALL=(ALL) NOPASSWD: /sbin/iptables' + validate: 'visudo -cf %s' + become: yes