From d7fb136d7d752aef8a100854f265d984f79c2ff0 Mon Sep 17 00:00:00 2001 From: Christian Franke Date: Fri, 4 Sep 2015 02:07:14 +0200 Subject: Use s3 to backup/restore git --- ancient | 2 ++ backup_ancient.sh | 2 ++ backup_git.yaml | 23 ++++++++++++++++++ deploy_staging.sh | 2 +- gitservers.yaml | 1 + roles/common/tasks/main.yaml | 3 ++- roles/gitserver/tasks/main.yaml | 2 +- roles/gitserver/tasks/restore_gitolite.yaml | 34 +++++++++++++++++++++------ roles/repo_service/tasks/as_repo_service.yaml | 5 ++++ roles/repo_service/tasks/main.yaml | 14 +++++++++++ roles/subdap/tasks/main.yaml | 1 - site.yaml | 2 ++ vars/aws.yaml | 14 +++++++++++ 13 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 ancient create mode 100755 backup_ancient.sh create mode 100644 backup_git.yaml create mode 100644 roles/repo_service/tasks/as_repo_service.yaml create mode 100644 roles/repo_service/tasks/main.yaml create mode 100644 vars/aws.yaml diff --git a/ancient b/ancient new file mode 100644 index 0000000..4bd5224 --- /dev/null +++ b/ancient @@ -0,0 +1,2 @@ +[gitservers] +oberon.sublab.org ansible_ssh_user=root diff --git a/backup_ancient.sh b/backup_ancient.sh new file mode 100755 index 0000000..2ea769b --- /dev/null +++ b/backup_ancient.sh @@ -0,0 +1,2 @@ +#!/bin/sh +ansible-playbook -i ancient --vault-password-file=~/.vault-pass.txt "$@" backup_git.yaml diff --git a/backup_git.yaml b/backup_git.yaml new file mode 100644 index 0000000..0a436df --- /dev/null +++ b/backup_git.yaml @@ -0,0 +1,23 @@ +- hosts: gitservers + vars_files: + - vars/aws.yaml + tasks: + - name: Ensure that boto is not installed via apt + apt: name=python-boto state=absent + - name: Ensure recent boto is installed + pip: name=boto state=present + - name: Create encrypted backup + shell: tar c repositories | gpg --symmetric --cipher-algo AES --passphrase "{{ aws_encryption_key }}" > repositories.tar.asc + args: + chdir: /var/lib/gitolite + - name: Upload backup to S3 + s3: + bucket=devops-sublab + object=git-backup.tar.asc + src=/var/lib/gitolite/repositories.tar.asc + mode=put + aws_access_key="{{aws_access_key}}" + aws_secret_key="{{aws_secret_key}}" + region=eu-central-1 + - name: Cleanup backup archive + file: path=/var/lib/gitolite/repositories.tar.asc state=absent diff --git a/deploy_staging.sh b/deploy_staging.sh index 017a52c..503eebf 100755 --- a/deploy_staging.sh +++ b/deploy_staging.sh @@ -1,2 +1,2 @@ #!/bin/sh -python2 $(which ansible-playbook) -i staging --vault-password-file=~/.vault-pass.txt "$@" site.yaml +ansible-playbook -i staging --vault-password-file=~/.vault-pass.txt "$@" site.yaml diff --git a/gitservers.yaml b/gitservers.yaml index 412cb04..c50cbbf 100644 --- a/gitservers.yaml +++ b/gitservers.yaml @@ -1,5 +1,6 @@ - hosts: gitservers roles: - common + - repo_service - gitserver - cgit diff --git a/roles/common/tasks/main.yaml b/roles/common/tasks/main.yaml index 3bcffb7..73b5039 100644 --- a/roles/common/tasks/main.yaml +++ b/roles/common/tasks/main.yaml @@ -1,8 +1,9 @@ --- # General setup for my hosts - name: Install common packages - apt: name={{ item }} state=present + apt: name={{ item }} state=present update_cache=yes with_items: + - git - htop - mtr - ntp diff --git a/roles/gitserver/tasks/main.yaml b/roles/gitserver/tasks/main.yaml index 7d1e3d7..aca95f4 100644 --- a/roles/gitserver/tasks/main.yaml +++ b/roles/gitserver/tasks/main.yaml @@ -1,6 +1,6 @@ --- - name: Install gitolite - apt: name={{ item }} + apt: name={{ item }} update_cache=yes with_items: - gitolite3 - git-daemon-run diff --git a/roles/gitserver/tasks/restore_gitolite.yaml b/roles/gitserver/tasks/restore_gitolite.yaml index c937ad1..78a9a0b 100644 --- a/roles/gitserver/tasks/restore_gitolite.yaml +++ b/roles/gitserver/tasks/restore_gitolite.yaml @@ -10,19 +10,34 @@ chdir: /var/lib/gitolite creates: /var/lib/gitolite/.gitolite/conf +#- name: Get repository backup +# get_url: +# url=https://oberon.sublab.org/protected/repositories.tar.gz +# url_username="{{download_user}}" +# url_password="{{download_pass}}" +# validate_certs=no +# dest=/var/lib/gitolite/repositories.tar.gz + - name: Get repository backup - get_url: - url=https://oberon.sublab.org/protected/repositories.tar.gz - url_username="{{download_user}}" - url_password="{{download_pass}}" - validate_certs=no - dest=/var/lib/gitolite/repositories.tar.gz + s3: + bucket=devops-sublab + object=git-backup.tar.asc + dest=/var/lib/gitolite/repositories.tar.asc + mode=get + aws_access_key="{{aws_access_key}}" + aws_secret_key="{{aws_secret_key}}" + region=eu-central-1 + +- name: Decrypt backup + shell: gpg --decrypt --passphrase "{{aws_encryption_key}}" repositories.tar.asc > repositories.tar + args: + chdir: /var/lib/gitolite - name: Create directory for backup unpack file: name=/var/lib/gitolite/old-repositories state=directory - name: Unpack repository backup - shell: tar -x --strip-components=1 -f ../repositories.tar.gz + shell: tar -x --strip-components=1 -f ../repositories.tar args: chdir: /var/lib/gitolite/old-repositories creates: /var/lib/gitolite/old-repositories/gitolite-admin.git @@ -42,3 +57,8 @@ shell: ./restore-backup.sh args: chdir: /var/lib/gitolite + +- name: Cleanup backup + shell: rm -rf repositories.tar.asc repositories.tar old-repositories + args: + chdir: /var/lib/gitolite diff --git a/roles/repo_service/tasks/as_repo_service.yaml b/roles/repo_service/tasks/as_repo_service.yaml new file mode 100644 index 0000000..6b364d8 --- /dev/null +++ b/roles/repo_service/tasks/as_repo_service.yaml @@ -0,0 +1,5 @@ +--- +- name: Clone repo_service git + git: + repo=https://github.com/cfra/repo_service.git + dest=/var/lib/repo_service/src diff --git a/roles/repo_service/tasks/main.yaml b/roles/repo_service/tasks/main.yaml new file mode 100644 index 0000000..b17319c --- /dev/null +++ b/roles/repo_service/tasks/main.yaml @@ -0,0 +1,14 @@ +--- +- name: Create repo_service group + group: name=repo_service + +- name: Create repo_service user + user: + name=repo_service + group=repo_service + home=/var/lib/repo_service + +- include: as_repo_service.yaml + become: yes + become_user: repo_service + become_method: su diff --git a/roles/subdap/tasks/main.yaml b/roles/subdap/tasks/main.yaml index 4d79aa1..5af24f6 100644 --- a/roles/subdap/tasks/main.yaml +++ b/roles/subdap/tasks/main.yaml @@ -1,7 +1,6 @@ --- - apt: name={{ item }} state=present update_cache=yes with_items: - - git - python-ldap - python-lxml - python-m2crypto diff --git a/site.yaml b/site.yaml index a0e7b28..b1b1cf9 100644 --- a/site.yaml +++ b/site.yaml @@ -1,8 +1,10 @@ --- - include: gitservers.yaml vars_files: + - vars/aws.yaml - vars/main.yaml - include: webservers.yaml vars_files: + - vars/aws.yaml - vars/main.yaml diff --git a/vars/aws.yaml b/vars/aws.yaml new file mode 100644 index 0000000..c5d340a --- /dev/null +++ b/vars/aws.yaml @@ -0,0 +1,14 @@ +$ANSIBLE_VAULT;1.1;AES256 +65356137373165336535376535336661316332306332633932383436633266383831313732663336 +3438356663353938643537353732353263383165353637350a393661316631313762303966393062 +37323133313061613630313631663637653564356538353430316432383731633766656534363434 +6636333365386538350a346366336264326264656661323037386462356665323033663433616635 +32323332393630396232663834346535373134316363303061656434613738336565386465623139 +62336530323035336566306633343462613161303937336266656532383134363034663632616564 +37396566333961623864353864313636326466613734663735303435356566303038643435626234 +36353032663633373264646337633361316535636238643932363433356336393164613363323239 +61666139613062663863383238343861396161343764636333343236363161646334383039353533 +37323135363065346532653066313966373963333336336664363439316634653565333339623734 +35343332363536313033346439623666333461626639616230643465366562383234313464656632 +35643331626265666462663637336235363466616136356432313835653366333238666363343261 +39613936633939653836383164633237636530363138323166336439373730326230 -- cgit v1.2.1