From 922ae13988923e7cb3dca55ddc2babd6e8644e84 Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Wed, 24 Jun 2020 12:00:06 +0300 Subject: [PATCH 1/8] Fix deployment for new users --- infra/ansible/deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infra/ansible/deploy.yml b/infra/ansible/deploy.yml index 8c76ed74a..81afd89fe 100644 --- a/infra/ansible/deploy.yml +++ b/infra/ansible/deploy.yml @@ -30,6 +30,9 @@ deploy_helper: "path={{ project_root }} release={{ project_release|default(omit) }} state=present shared_path={{ project_shared_path }}" ### Update source + - name: Ensure root source dir is present + file: "path={{ project_source_path }} state=directory" + - name: Download remote package get_url: url: "{{ project_package }}" -- GitLab From 0e02298e7f580d763102704459e5c426cb634150 Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Wed, 24 Jun 2020 15:47:50 +0300 Subject: [PATCH 2/8] Composite deployment without backups --- Jenkinsfile | 2 +- infra/ansible/ansible.cfg | 1 + infra/ansible/deploy.yml | 43 +++++++++++++++++-- infra/ansible/inventory/dev1.ini | 5 +++ infra/ansible/inventory/dev1.yml | 2 - infra/ansible/inventory/dev2.ini | 5 +++ infra/ansible/inventory/dev2.yml | 2 - .../ansible/inventory/{prod.yml => prod.ini} | 0 .../inventory/{stage1.yml => stage1.ini} | 0 .../inventory/{stage2.yml => stage2.ini} | 0 infra/ansible/setup.yml | 13 ++++++ 11 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 infra/ansible/inventory/dev1.ini delete mode 100644 infra/ansible/inventory/dev1.yml create mode 100644 infra/ansible/inventory/dev2.ini delete mode 100644 infra/ansible/inventory/dev2.yml rename infra/ansible/inventory/{prod.yml => prod.ini} (100%) rename infra/ansible/inventory/{stage1.yml => stage1.ini} (100%) rename infra/ansible/inventory/{stage2.yml => stage2.ini} (100%) create mode 100644 infra/ansible/setup.yml diff --git a/Jenkinsfile b/Jenkinsfile index 094cb8061..1cf519522 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -158,7 +158,7 @@ def deploy(String envname) { usernameVariable: 'JENKINS_ERL_CI_USER')]) { sh 'cd infra/ansible && ansible-playbook \ - -i inventory/' + envname + '.yml \ + -i inventory/' + envname + '.ini \ -e project_package=${DEPLOY_PACKAGE} \ -e project_release=${DEPLOY_RELEASE} \ -u ${JENKINS_ERL_CI_USER} \ diff --git a/infra/ansible/ansible.cfg b/infra/ansible/ansible.cfg index a6ab924c2..279cc76ae 100644 --- a/infra/ansible/ansible.cfg +++ b/infra/ansible/ansible.cfg @@ -1,6 +1,7 @@ [defaults] host_key_checking = False timeout = 30 +interpreter_python = auto_silent [ssh_connection] ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=1800s diff --git a/infra/ansible/deploy.yml b/infra/ansible/deploy.yml index 81afd89fe..582a79139 100644 --- a/infra/ansible/deploy.yml +++ b/infra/ansible/deploy.yml @@ -23,6 +23,9 @@ - path: log src: log project_api_port: 8888 + project_mqtt_ports: [8083,8084,8883,1883] + # Can be set in the inventory variables or CLI + project_drop_mqtt_traffic: "{{ drop_mqtt_traffic | default(true) }}" tasks: ### Initialize @@ -39,18 +42,15 @@ dest: "{{ project_source_path }}/{{ project_package|basename }}" checksum: "{{ project_package_checksum|default(omit) }}" when: "'http' in project_package" - # register: remote_package - name: Upload local package copy: src: "{{ project_package }}" dest: "{{ project_source_path }}/{{ project_package|basename }}" when: "'http' not in project_package" - # register: local_package - name: Ensure sources dir is present file: "path={{ project_source_path }}/{{ deploy_helper.new_release }} state=directory" - # when: remote_package.changed or local_package.changed register: new_src - name: Extract the package @@ -106,10 +106,34 @@ register: ping when: binary.stat.exists == True + # Prepare for fast start + - name: Clean MQTT subscription tables + command: + cmd: "{{ deploy_helper.current_path }}/bin/server eval '[ mnesia:clear_table(X) || X <- [mqtt_subscription, mqtt_subscriber, mqtt_subproperty]]'" + when: binary.stat.exists == True and ping.stdout == "pong" + + - name: Dump Mnesia log + command: + cmd: "{{ deploy_helper.current_path }}/bin/server eval 'mnesia:dump_log()'" + when: binary.stat.exists == True and ping.stdout == "pong" + - name: Stop erlang server command: "{{ deploy_helper.current_path }}/bin/server stop" when: binary.stat.exists == True and ping.stdout == "pong" + - name: Drop MQTT traffic + iptables: + chain: INPUT + action: insert + protocol: tcp + destination_port: "{{ item | int }}" + jump: DROP + state: present + with_items: "{{ project_mqtt_ports }}" + become: yes + become_user: root + when: project_drop_mqtt_traffic | bool + - name: Finalize the deploy deploy_helper: path={{ project_root }} release={{ deploy_helper.new_release }} state=finalize clean={{ project_clean|bool }} keep_releases={{ project_keep_releases|int }} @@ -132,3 +156,16 @@ timeout: 10 status_code: 200 tags: [health-check] + + - name: Allow MQTT traffic + iptables: + chain: INPUT + action: insert + protocol: tcp + destination_port: "{{ item | int }}" + jump: DROP + state: absent + with_items: "{{ project_mqtt_ports }}" + become: yes + become_user: root + when: project_drop_mqtt_traffic | bool diff --git a/infra/ansible/inventory/dev1.ini b/infra/ansible/inventory/dev1.ini new file mode 100644 index 000000000..b8ec3c5d8 --- /dev/null +++ b/infra/ansible/inventory/dev1.ini @@ -0,0 +1,5 @@ +[appservers] +35.227.151.129 + +[appservers:vars] +drop_mqtt_traffic=false diff --git a/infra/ansible/inventory/dev1.yml b/infra/ansible/inventory/dev1.yml deleted file mode 100644 index f1b5e4364..000000000 --- a/infra/ansible/inventory/dev1.yml +++ /dev/null @@ -1,2 +0,0 @@ -[appservers] -35.227.151.129 diff --git a/infra/ansible/inventory/dev2.ini b/infra/ansible/inventory/dev2.ini new file mode 100644 index 000000000..a3c53b7da --- /dev/null +++ b/infra/ansible/inventory/dev2.ini @@ -0,0 +1,5 @@ +[appservers] +35.247.90.7 + +[appservers:vars] +; drop_mqtt_traffic=false diff --git a/infra/ansible/inventory/dev2.yml b/infra/ansible/inventory/dev2.yml deleted file mode 100644 index a67f035e0..000000000 --- a/infra/ansible/inventory/dev2.yml +++ /dev/null @@ -1,2 +0,0 @@ -[appservers] -35.247.90.7 diff --git a/infra/ansible/inventory/prod.yml b/infra/ansible/inventory/prod.ini similarity index 100% rename from infra/ansible/inventory/prod.yml rename to infra/ansible/inventory/prod.ini diff --git a/infra/ansible/inventory/stage1.yml b/infra/ansible/inventory/stage1.ini similarity index 100% rename from infra/ansible/inventory/stage1.yml rename to infra/ansible/inventory/stage1.ini diff --git a/infra/ansible/inventory/stage2.yml b/infra/ansible/inventory/stage2.ini similarity index 100% rename from infra/ansible/inventory/stage2.yml rename to infra/ansible/inventory/stage2.ini diff --git a/infra/ansible/setup.yml b/infra/ansible/setup.yml new file mode 100644 index 000000000..b50b13856 --- /dev/null +++ b/infra/ansible/setup.yml @@ -0,0 +1,13 @@ +--- +- name: Setup app servers + hosts: appservers + + tasks: + - name: Allow 'nynja' user to run iptables + lineinfile: + dest: /etc/sudoers + state: present + regexp: '^nynja' + line: 'nynja ALL=(ALL) NOPASSWD: /sbin/iptables' + validate: 'visudo -cf %s' + become: yes -- GitLab From 1c3dd3d34409fb97b96745454028ec0612e50a05 Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Wed, 24 Jun 2020 17:45:43 +0300 Subject: [PATCH 3/8] Add backups support to deploy playbook --- infra/ansible/deploy.yml | 36 +++++++++++++++++++++++++- infra/ansible/files/backup.sh | 44 ++++++++++++++++++++++++++++++++ infra/ansible/inventory/dev2.ini | 2 ++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100755 infra/ansible/files/backup.sh diff --git a/infra/ansible/deploy.yml b/infra/ansible/deploy.yml index 582a79139..1b52a1a9e 100644 --- a/infra/ansible/deploy.yml +++ b/infra/ansible/deploy.yml @@ -3,7 +3,9 @@ hosts: appservers vars: - project_root: "/home/nynja/erlang-server" + project_home: "/home/nynja" + project_root: "{{ project_home}}/erlang-server" + project_backups_dir: "{{ backups_dir | default('/home/nynja/backups') }}" project_shared_path: "{{ project_root }}/shared" project_source_path: "{{ project_root }}/shared/source" project_database_name: "Mnesia.mq2@{{ ansible_fqdn }}" @@ -26,6 +28,8 @@ project_mqtt_ports: [8083,8084,8883,1883] # Can be set in the inventory variables or CLI project_drop_mqtt_traffic: "{{ drop_mqtt_traffic | default(true) }}" + project_auto_backups: "{{ auto_backups | default(false) }}" + project_deploy_backups: "{{ deploy_backups | default(false) }}" tasks: ### Initialize @@ -94,6 +98,27 @@ file: "path='{{ deploy_helper.new_release_path }}/{{ item.path }}' src='{{ deploy_helper.shared_path }}/{{ item.src }}' state=link" with_items: "{{ project_shared_children }}" + ### Setup backups + - name: Make sure ~/bin exists + file: + path: "{{ project_home }}/bin" + state: directory + mode: '0755' + + - name: Install backup script + copy: + src: files/backup.sh + dest: "{{ project_home }}/bin/backup.sh" + mode: '0700' + + - name: Setup backups cronjob + cron: + name: "backup erlang server database" + minute: "0" + hour: "*/6" + job: "{{ project_home }}/bin/backup.sh {{ deploy_helper.current_path }} {{ project_backups_dir }}" + state: "{{ 'present' if project_auto_backups else 'absent' }}" + ### Finalize - name: Check server binary stat: @@ -106,6 +131,15 @@ register: ping when: binary.stat.exists == True + # Create a backup + - name: Create pre-deploy backup + command: + cmd: "{{ project_home }}/bin/backup.sh {{ deploy_helper.current_path }} {{ project_backups_dir }}" + when: + - binary.stat.exists == True + - ping.stdout == "pong" + - project_deploy_backups|bool + # Prepare for fast start - name: Clean MQTT subscription tables command: diff --git a/infra/ansible/files/backup.sh b/infra/ansible/files/backup.sh new file mode 100755 index 000000000..8fb3be061 --- /dev/null +++ b/infra/ansible/files/backup.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e + +BACKUP_TIME=`date '+%a_%d_%b_%Y_%H_%M_%S'` +DATE=`date '+%Y-%m-%dT%H'` +SERVER_DIR=${1:?"Server directory must be provided as first argument"} +BACKUP_DIR=${2:?"Backup directory must be provided as second argument"} +BACKUP_HOST=`hostname -A | xargs` +BACKUP_FILE_REL="mq2@${BACKUP_HOST}_bkp_${DATE}*" +BACKUP_FILE="${SERVER_DIR}/backup/${BACKUP_FILE_REL}" +BACKUP_LOG="${BACKUP_DIR}/log/backup_nodetool.log" + + +mkdir -p ${BACKUP_DIR} +mkdir -p ${BACKUP_DIR}/log + +echo "Deleting old backupfile. Rention Time is 30 days." >> ${BACKUP_LOG} +find ${BACKUP_DIR} -name "mq*" -type f -mtime +30 -print -delete >> ${BACKUP_LOG} + + +echo "Backup Time: ${BACKUP_TIME}" >> ${BACKUP_LOG} + +${SERVER_DIR}/bin/server eval 'roster_db:backup()' + +echo "Backup File:" +ls -ltr ${BACKUP_FILE} + +echo "Moving backup file to ${BACKUP_DIR}/" >> ${BACKUP_LOG} +mv ${BACKUP_FILE} ${BACKUP_DIR}/ >> ${BACKUP_LOG} + +echo "Compressing backup file" >> ${BACKUP_LOG} +cd ${BACKUP_DIR} +tar -cJf mq2@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz ${BACKUP_FILE_REL} >> ${BACKUP_LOG} +rm -rf ${BACKUP_DIR}/${BACKUP_FILE_REL} >> ${BACKUP_LOG} + +echo "Compressed Backup File:" >> ${BACKUP_LOG} +ls -ltr ${BACKUP_DIR}/mq2@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz >> ${BACKUP_LOG} + +echo "Log File:" >> ${BACKUP_LOG} +ls -ltr ${BACKUP_LOG} >> ${BACKUP_LOG} + +echo "Job Completed" >> ${BACKUP_LOG} +echo "--------------------------------" >> ${BACKUP_LOG} diff --git a/infra/ansible/inventory/dev2.ini b/infra/ansible/inventory/dev2.ini index a3c53b7da..2a9c72542 100644 --- a/infra/ansible/inventory/dev2.ini +++ b/infra/ansible/inventory/dev2.ini @@ -3,3 +3,5 @@ [appservers:vars] ; drop_mqtt_traffic=false +; auto_backups=true +; deploy_backups=true -- GitLab From ef020dd03f08174c2a56a89b5951483f10bfad40 Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Wed, 24 Jun 2020 17:47:29 +0300 Subject: [PATCH 4/8] Configure environment variables --- infra/ansible/inventory/dev1.ini | 3 --- infra/ansible/inventory/dev2.ini | 5 ----- infra/ansible/inventory/prod.ini | 5 +++++ infra/ansible/inventory/stage1.ini | 5 +++++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/infra/ansible/inventory/dev1.ini b/infra/ansible/inventory/dev1.ini index b8ec3c5d8..f1b5e4364 100644 --- a/infra/ansible/inventory/dev1.ini +++ b/infra/ansible/inventory/dev1.ini @@ -1,5 +1,2 @@ [appservers] 35.227.151.129 - -[appservers:vars] -drop_mqtt_traffic=false diff --git a/infra/ansible/inventory/dev2.ini b/infra/ansible/inventory/dev2.ini index 2a9c72542..a67f035e0 100644 --- a/infra/ansible/inventory/dev2.ini +++ b/infra/ansible/inventory/dev2.ini @@ -1,7 +1,2 @@ [appservers] 35.247.90.7 - -[appservers:vars] -; drop_mqtt_traffic=false -; auto_backups=true -; deploy_backups=true diff --git a/infra/ansible/inventory/prod.ini b/infra/ansible/inventory/prod.ini index 4403aa463..ef10df01b 100644 --- a/infra/ansible/inventory/prod.ini +++ b/infra/ansible/inventory/prod.ini @@ -1,2 +1,7 @@ [appservers] 35.230.49.68 + +[appservers:vars] +drop_mqtt_traffic=true +auto_backups=true +deploy_backups=true diff --git a/infra/ansible/inventory/stage1.ini b/infra/ansible/inventory/stage1.ini index 18a1228ef..4b8eed439 100644 --- a/infra/ansible/inventory/stage1.ini +++ b/infra/ansible/inventory/stage1.ini @@ -1,2 +1,7 @@ [appservers] 35.197.66.129 + +[appservers:vars] +drop_mqtt_traffic=true +auto_backups=true +deploy_backups=true -- GitLab From c6008dc142326e1917f91fa3a49b2931692a68ae Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Thu, 25 Jun 2020 12:20:55 +0300 Subject: [PATCH 5/8] Use DB name variable in backup script --- infra/ansible/files/backup.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/infra/ansible/files/backup.sh b/infra/ansible/files/backup.sh index 8fb3be061..dc88ec465 100755 --- a/infra/ansible/files/backup.sh +++ b/infra/ansible/files/backup.sh @@ -6,8 +6,9 @@ BACKUP_TIME=`date '+%a_%d_%b_%Y_%H_%M_%S'` DATE=`date '+%Y-%m-%dT%H'` SERVER_DIR=${1:?"Server directory must be provided as first argument"} BACKUP_DIR=${2:?"Backup directory must be provided as second argument"} +BACKUP_DBNAME="mq2" BACKUP_HOST=`hostname -A | xargs` -BACKUP_FILE_REL="mq2@${BACKUP_HOST}_bkp_${DATE}*" +BACKUP_FILE_REL="${BACKUP_DBNAME}@${BACKUP_HOST}_bkp_${DATE}*" BACKUP_FILE="${SERVER_DIR}/backup/${BACKUP_FILE_REL}" BACKUP_LOG="${BACKUP_DIR}/log/backup_nodetool.log" @@ -31,11 +32,11 @@ mv ${BACKUP_FILE} ${BACKUP_DIR}/ >> ${BACKUP_LOG} echo "Compressing backup file" >> ${BACKUP_LOG} cd ${BACKUP_DIR} -tar -cJf mq2@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz ${BACKUP_FILE_REL} >> ${BACKUP_LOG} +tar -cJf ${BACKUP_DBNAME}@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz ${BACKUP_FILE_REL} >> ${BACKUP_LOG} rm -rf ${BACKUP_DIR}/${BACKUP_FILE_REL} >> ${BACKUP_LOG} echo "Compressed Backup File:" >> ${BACKUP_LOG} -ls -ltr ${BACKUP_DIR}/mq2@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz >> ${BACKUP_LOG} +ls -ltr ${BACKUP_DIR}/${BACKUP_DBNAME}@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz >> ${BACKUP_LOG} echo "Log File:" >> ${BACKUP_LOG} ls -ltr ${BACKUP_LOG} >> ${BACKUP_LOG} -- GitLab From 5edc6cd057eca5f3f38f0beb48c0125a04134c28 Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Thu, 25 Jun 2020 13:23:08 +0300 Subject: [PATCH 6/8] Drop MQTT traffic before table cleanup --- infra/ansible/deploy.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/infra/ansible/deploy.yml b/infra/ansible/deploy.yml index 1b52a1a9e..319ac3237 100644 --- a/infra/ansible/deploy.yml +++ b/infra/ansible/deploy.yml @@ -140,21 +140,11 @@ - ping.stdout == "pong" - project_deploy_backups|bool - # Prepare for fast start - - name: Clean MQTT subscription tables - command: - cmd: "{{ deploy_helper.current_path }}/bin/server eval '[ mnesia:clear_table(X) || X <- [mqtt_subscription, mqtt_subscriber, mqtt_subproperty]]'" - when: binary.stat.exists == True and ping.stdout == "pong" - - name: Dump Mnesia log command: cmd: "{{ deploy_helper.current_path }}/bin/server eval 'mnesia:dump_log()'" when: binary.stat.exists == True and ping.stdout == "pong" - - name: Stop erlang server - command: "{{ deploy_helper.current_path }}/bin/server stop" - when: binary.stat.exists == True and ping.stdout == "pong" - - name: Drop MQTT traffic iptables: chain: INPUT @@ -168,6 +158,15 @@ become_user: root when: project_drop_mqtt_traffic | bool + - name: Clean MQTT subscription tables + command: + cmd: "{{ deploy_helper.current_path }}/bin/server eval '[ mnesia:clear_table(X) || X <- [mqtt_subscription, mqtt_subscriber, mqtt_subproperty]]'" + when: binary.stat.exists == True and ping.stdout == "pong" + + - name: Stop erlang server + command: "{{ deploy_helper.current_path }}/bin/server stop" + when: binary.stat.exists == True and ping.stdout == "pong" + - name: Finalize the deploy deploy_helper: path={{ project_root }} release={{ deploy_helper.new_release }} state=finalize clean={{ project_clean|bool }} keep_releases={{ project_keep_releases|int }} -- GitLab From 160c1f5c6f542a3dedc879342b93db35c3f98f51 Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Thu, 25 Jun 2020 13:32:14 +0300 Subject: [PATCH 7/8] Set proper backups permissions --- infra/ansible/files/backup.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/ansible/files/backup.sh b/infra/ansible/files/backup.sh index dc88ec465..e4c20b3ae 100755 --- a/infra/ansible/files/backup.sh +++ b/infra/ansible/files/backup.sh @@ -10,6 +10,7 @@ BACKUP_DBNAME="mq2" BACKUP_HOST=`hostname -A | xargs` BACKUP_FILE_REL="${BACKUP_DBNAME}@${BACKUP_HOST}_bkp_${DATE}*" BACKUP_FILE="${SERVER_DIR}/backup/${BACKUP_FILE_REL}" +BACKUP_ARCHIVE=${BACKUP_DBNAME}@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz BACKUP_LOG="${BACKUP_DIR}/log/backup_nodetool.log" @@ -32,11 +33,12 @@ mv ${BACKUP_FILE} ${BACKUP_DIR}/ >> ${BACKUP_LOG} echo "Compressing backup file" >> ${BACKUP_LOG} cd ${BACKUP_DIR} -tar -cJf ${BACKUP_DBNAME}@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz ${BACKUP_FILE_REL} >> ${BACKUP_LOG} +tar -cJf ${BACKUP_ARCHIVE} ${BACKUP_FILE_REL} >> ${BACKUP_LOG} +chmod 640 ${BACKUP_ARCHIVE} rm -rf ${BACKUP_DIR}/${BACKUP_FILE_REL} >> ${BACKUP_LOG} echo "Compressed Backup File:" >> ${BACKUP_LOG} -ls -ltr ${BACKUP_DIR}/${BACKUP_DBNAME}@${BACKUP_HOST}_${BACKUP_TIME}.tar.xz >> ${BACKUP_LOG} +ls -ltr ${BACKUP_DIR}/${BACKUP_ARCHIVE} >> ${BACKUP_LOG} echo "Log File:" >> ${BACKUP_LOG} ls -ltr ${BACKUP_LOG} >> ${BACKUP_LOG} -- GitLab From 1f7914d4fe04c69f698684aac23e2206a9dca30d Mon Sep 17 00:00:00 2001 From: Dincho Todorov Date: Thu, 25 Jun 2020 13:36:20 +0300 Subject: [PATCH 8/8] Set non-default backup dirs for stage and prod --- infra/ansible/inventory/prod.ini | 1 + infra/ansible/inventory/stage1.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/infra/ansible/inventory/prod.ini b/infra/ansible/inventory/prod.ini index ef10df01b..538adc38e 100644 --- a/infra/ansible/inventory/prod.ini +++ b/infra/ansible/inventory/prod.ini @@ -5,3 +5,4 @@ drop_mqtt_traffic=true auto_backups=true deploy_backups=true +backups_dir=/backup/erlang-server diff --git a/infra/ansible/inventory/stage1.ini b/infra/ansible/inventory/stage1.ini index 4b8eed439..a716bf19b 100644 --- a/infra/ansible/inventory/stage1.ini +++ b/infra/ansible/inventory/stage1.ini @@ -5,3 +5,4 @@ drop_mqtt_traffic=true auto_backups=true deploy_backups=true +backups_dir=/backup/erlang-server -- GitLab