1
0
Fork 0
mirror of https://gitlab.com/s3lph/ansible-collection-dirvish synced 2024-12-04 15:11:02 +01:00

Initial commit

This commit is contained in:
s3lph 2020-10-31 04:34:41 +01:00
commit cadfc11ec5
17 changed files with 862 additions and 0 deletions

5
README.md Normal file
View file

@ -0,0 +1,5 @@
# Ansible Collection - s3lph.dirvish
Documentation for the collection.
WIP, TBD

60
galaxy.yml Normal file
View file

@ -0,0 +1,60 @@
# The namespace of the collection. This can be a company/brand/organization or product namespace under which all
# content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with
# underscores or numbers and cannot contain consecutive underscores
namespace: s3lph
# The name of the collection. Has the same character restrictions as 'namespace'
name: dirvish
# The version of the collection. Must be compatible with semantic versioning
version: 0.5
# The path to the Markdown (.md) readme file. This path is relative to the root of the collection
readme: README.md
# A list of the collection's content authors. Can be just the name or in the format 'Full Name <email> (url)
# @nicks:irc/im.site#channel'
authors:
- s3lph <account-gitlab-ideynizv@kernelpanic.lol>
### OPTIONAL but strongly recommended
# A short summary description of the collection
description: Server and client configuration for dirvish backups
# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only
# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file'
license:
- MIT
# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character
# requirements as 'namespace' and 'name'
tags:
- dirvish
- backup
- rsync
# Collections that this collection requires to be installed for it to be usable. The key of the dict is the
# collection label 'namespace.name'. The value is a version range
# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
# range specifiers can be set and are separated by ','
dependencies: {}
# The URL of the originating SCM repository
repository: https://gitlab.com/s3lph/ansible-collection-dirvish
# The URL to any online docs
documentation: https://gitlab.com/s3lph/ansible-collection-dirvish
# The URL to the homepage of the collection/project
homepage: http://example.com
# The URL to the collection issue tracker
issues: https://gitlab.com/s3lph/ansible-collection-dirvish/-/issues
# A list of file glob-like patterns used to filter any files or directories that should not be included in the build
# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This
# uses 'fnmatch' to match the files or directories. Some directories and files like 'galaxy.yml', '*.pyc', '*.retry',
# and '.git' are always filtered
build_ignore: []

View file

@ -0,0 +1,24 @@
---
dirvish_client_cronjob_directory: /etc/cron.d
dirvish_client_snapshot_directory: /var/lib/dirvish_client
dirvish_client_snapshot_scripts: []
dirvish_client_snapshot_cronexpr: "0 3 * * *"
### SNAPSHOT DEFAULTS
# Gitea
dirvish_client_gitea_backupdir: /var/backup/gitea
dirvish_client_gitea_configfile: /etc/gitea/app.ini
dirvish_client_gitea_bin: /usr/bin/gitea
dirvish_client_gitea_user: gitea
dirvish_client_gitea_group: gitea
# InfluxDB
dirvish_client_influxdb_backupdir: /var/backup/influxdb
dirvish_client_influxdb_host: "localhost:8088"
dirvish_client_influxdb_bin: /usr/bin/influxd
# MariaDB
dirvish_client_mysql_backupdir: /var/backup/mysql
dirvish_clieny_mysql_mycnf: /root/.mysql_backup_config.cnf

View file

@ -0,0 +1,8 @@
---
- name: install depdencies
package:
name: "{{ item }}"
state: present
loop:
- rsync

View file

@ -0,0 +1,13 @@
---
- name: install dependencies
import_tasks: dependencies.yml
tags:
- "role::dirvish_client"
- "role::dirvish_client:dependencies"
- name: install snapshot scripts
import_tasks: snapshots.yml
tags:
- "role::dirvish_client"
- "role::dirvish_client:snapshots"

View file

@ -0,0 +1,34 @@
---
- name: create snapshot scripts directory
file:
path: "{{ dirvish_client_snapshot_directory }}"
state: directory
owner: root
group: root
mode: "0755"
- name: install snapshot scripts
template:
src: "snapshot_scripts/{{ item }}.j2"
dest: "{{ dirvish_client_snapshot_directory }}/{{ item }}"
owner: root
group: root
mode: "0700"
loop: "{{ dirvish_client_snapshot_scripts }}"
- name: install snapshot master script
template:
src: dirvish-pre-client-snapshot.sh.j2
dest: /usr/local/bin/dirvish-pre-client-snapshot.sh
owner: root
group: root
mode: "0755"
- name: install snapshot master cronjob
template:
src: dirvish-pre-client-snapshot.cronjob.j2
dest: "{{ dirvish_client_cronjob_directory }}/dirvish-pre-client-snapshot"
owner: root
group: root
mode: "0644"

View file

@ -0,0 +1,3 @@
{{ ansible_managed | comment }}
{{ dirvish_client_snapshot_cronexpr }} root /usr/local/bin/dirvish-pre-client-snapshot.sh

View file

@ -0,0 +1,12 @@
#!/bin/bash
#
{{ ansible_managed | comment }}
set -e
mkdir -p /var/backup
{% for snapshot_script in dirvish_client_snapshot_scripts %}
{{ dirvish_client_snapshot_directory }}/{{ snapshot_script }}
{% endfor %}
echo $(date +%s) > /var/backup/.dirvish-snapshot-success

View file

@ -0,0 +1,30 @@
---
dirvish_server_master_conf: /etc/dirvish/master.conf
dirvish_server_cronjob: /etc/cron.d/dirvish
dirvish_server_cronjob_hour_initial: 4
dirvish_server_cronjob_minute_initial: 0
dirvish_server_cronjob_step_minutes: 5
dirvish_server_cronjob_dom: "*"
dirvish_server_cronjob_month: "*"
dirvish_server_cronjob_dow: "*"
dirvish_server_bank_mounts: [/]
dirvish_server_clients_group: clients
dirvish_server_rsh: /usr/bin/ssh
dirvish_server_branch_default: root
dirvish_server_image_name: "%Y%m%d-%H%M%S"
dirvish_server_log_compression: gzip
dirvish_server_index_compression: gzip
dirvish_server_expire_default: "+31 days"
dirvish_server_exclude:
- lost+found/
- dev/
- proc/
- run/
- sys/
- tmp/
dirvish_server_client_exclude: []

View file

@ -0,0 +1,51 @@
---
- name: create /etc/dirvish directory
file:
path: "{{ dirvish_server_master_conf | dirname }}"
state: directory
mode: "0755"
owner: root
group: root
- name: render dirvish master config
template:
src: master.conf.j2
dest: "{{ dirvish_server_master_conf }}"
mode: "0644"
owner: root
group: root
# ansible_mounts is broken - https://github.com/ansible/ansible/issues/24644
- name: get all mounts
shell: "cat /etc/mtab | cut -d' ' -f2" # noqa 306
changed_when: no
check_mode: no
register: dirvish_server_register_mtab
- name: make sure all banks are mounted
assert:
that: "item in mounts"
loop: "{{ dirvish_server_bank_mounts }}"
vars:
mounts: "{{ dirvish_server_register_mtab.stdout_lines }}"
- name: create all client directories
file:
path: "{{ hostvars[item].dirvish_server_bank }}/{{ item }}/dirvish"
state: directory
loop: "{{ groups[dirvish_server_clients_group] }}"
- name: render all client configs
template:
src: client_default.conf.j2
dest: "{{ hostvars[item].dirvish_server_bank }}/{{ item }}/dirvish/default.conf"
loop: "{{ groups[dirvish_server_clients_group] }}"
- name: render dirvish crontab
template:
src: dirvish.crontab.j2
dest: "{{ dirvish_server_cronjob }}"
mode: "0755"
owner: root
group: root

View file

@ -0,0 +1,18 @@
---
- name: install dependencies
package:
name: "{{ item }}"
state: present
loop:
- dirvish
- rsync
- openssh-client
- name: install modified dirvish-expire
template:
src: dirvish-expire.j2
dest: /usr/local/bin/dirvish-expire-configfile
owner: root
group: root
mode: "0755"

View file

@ -0,0 +1,20 @@
---
- name: install dependencies
import_tasks: dependencies.yml
tags:
- "role::dirvish_server"
- "role::dirvish_server:dependencies"
- name: configure dirvish
import_tasks: config.yml
tags:
- "role::dirvish_server"
- "role::dirvish_server:config"
- name: create initial non-incremental image
import_tasks: vault_init.yml
when: "dirvish_server_vault_init_host is defined"
tags:
- "role::dirvish_server"
- "role::dirvish_server:vault_init"

View file

@ -0,0 +1,15 @@
---
- name: run snapshot master script
delegate_to: "{{ dirvish_server_vault_init_host }}"
run_once: yes
command: /usr/local/bin/dirvish-pre-client-snapshot.sh
changed_when: yes
- name: create initial non-incremental image
command:
args:
cmd: "/usr/sbin/dirvish --config {{ dirvish_server_master_conf }} --vault {{ dirvish_server_vault_init_host }} --init"
creates: "{{ bank }}/{{ dirvish_server_vault_init_host }}/dirvish/{{ dirvish_server_branch_default }}.hist"
vars:
bank: "{{ hostvars[dirvish_server_vault_init_host].dirvish_server_bank }}"

View file

@ -0,0 +1,18 @@
{{ ansible_managed | comment }}
{% if item == inventory_hostname %}
client: localhost
{% else %}
client: {{ hostvars[item].dirvish_server_client_hostname | default(item) }}
{% endif %}
tree: /
{% if item != inventory_hostname %}
rsh: {{ hostvars[item].dirvish_server_client_rsh | default(dirvish_server_rsh) }}
{% endif %}
{% if hostvars[item].dirvish_server_client_exclude is defined and hostvars[item].dirvish_server_client_exclude | length > 0 %}
exclude:
{% for exclude in hostvars[item].dirvish_server_client_exclude %}
{{ exclude }}
{% endfor %}
{% endif %}

View file

@ -0,0 +1,510 @@
#!/usr/bin/perl
{{ ansible_managed | comment }}
# This is a slightly modified version of the original dirvish-expire
# script residing in /usr/sbin/dirvish-expire. Apart from the
# modifications listed below, copyright is attributed to the original
# authors as listed in the copyright statement below
#
# Modifications:
# 2020-10-15 s3lph Add --config command line flag
$CONFDIR = "/etc/dirvish";
# $Id: dirvish-expire.pl,v 12.0 2004/02/25 02:42:14 jw Exp $ $Name: Dirvish-1_2 $
$VERSION = ('$Name: Dirvish-1_2 $' =~ /Dirvish/i)
? ('$Name: Dirvish-1_2 $' =~ m/^.*:\s+dirvish-(.*)\s*\$$/i)[0]
: '1.1.2 patch' . ('$Id: dirvish-expire.pl,v 12.0 2004/02/25 02:42:14 jw Exp $'
=~ m/^.*,v(.*:\d\d)\s.*$/)[0];
$VERSION =~ s/_/./g;
#########################################################################
# #
# Copyright 2002 and $Date: 2004/02/25 02:42:14 $
# Pegasystems Technologies and J.W. Schultz #
# #
# Licensed under the Open Software License version 2.0 #
# #
# This program is free software; you can redistribute it #
# and/or modify it under the terms of the Open Software #
# License, version 2.0 by Lauwrence E. Rosen. #
# #
# This program is distributed in the hope that it will be #
# useful, but WITHOUT ANY WARRANTY; without even the implied #
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR #
# PURPOSE. See the Open Software License for details. #
# #
#########################################################################
use Time::ParseDate;
use POSIX qw(strftime);
use File::Find;
use Getopt::Long;
sub loadconfig;
sub check_expire;
sub findop;
sub imsort;
sub seppuku;
sub usage
{
my $message = shift(@_);
length($message) and print STDERR $message, "\n\n";
print STDERR <<EOUSAGE;
USAGE
dirvish.expire OPTIONS
OPTIONS
--time date_expression
--[no]tree
--config config_file
--vault vault_name
--no-run
--quiet
EOUSAGE
exit 255;
}
$Options =
{
help => \&usage,
version => sub {
print STDERR "dirvish version $VERSION\n";
exit(0);
},
};
GetOptions($Options, qw(
quiet!
config=s
vault=s
time=s
tree!
no-run|dry-run
version
help|?
)) or usage;
if ($$Options{config}) {
loadconfig(undef, $$Options{config}, $Options);
} else {
if ($CONFDIR =~ /dirvish$/ && -f "$CONFDIR.conf") {
loadconfig(undef, "$CONFDIR.conf", $Options);
} elsif (-f "$CONFDIR/master.conf") {
loadconfig(undef, "$CONFDIR/master.conf", $Options);
} elsif (-f "$CONFDIR/dirvish.conf") {
seppuku 250, <<EOERR;
ERROR: no master configuration file.
An old $CONFDIR/dirvish.conf file found.
Please read the dirvish release notes.
EOERR
} else {
seppuku 251, "ERROR: no global configuration file";
}
}
ref($$Options{vault}) and $$Options{vault} = undef;
$$Options{time} and $expire_time = parsedate($$Options{time});
$expire_time ||= time;
if ($$Options{vault})
{
my $b;
my $bank;
for $b (@{$$Options{bank}})
{
if (-d "$b/$$Options{vault}")
{
$bank = $b;
last;
}
}
$bank or seppuku 252, "Cannot find vault $$Options{vault}";
find(\&findop, join('/', $bank, $$Options{vault}));
} else {
for $bank (@{$$Options{bank}})
{
find(\&findop, $bank);
}
}
scalar(@expires) or exit 0;
if (!$$Options{quiet})
{
printf "Expiring images as of %s\n",
strftime('%Y-%m-%d %H:%M:%S', localtime($expire_time));
$$Options{vault} and printf "Restricted to vault %s\n",
$$Options{vault};
print "\n";
printf "%-15s %-15s %-16.16s %s\n",
qw(VAULT:BRANCH IMAGE CREATED EXPIRED);
}
for $expire (sort {imsort()} @expires)
{
my ($created, $expired);
($created = $$expire{created}) =~ s/:\d\d$//;
($expired = $$expire{expire}) =~ s/:\d\d$//;
if (!$unexpired{$$expire{vault}}{$$expire{branch}})
{
printf "cannot expire %s:%s:%s No unexpired good images\n",
$$expire{vault},
$$expire{branch},
$$expire{image};
$$expire{status} =~ /^success/
and ++$unexpired{$$expire{vault}}{$$expire{branch}};
# By virtue of the sort order this will be the newest
# image so that older ones can be expired.
next;
}
$$Options{quiet} or printf "%-15s %-15s %-16.16s %s\n",
$$expire{vault} . ':' . $$expire{branch},
$$expire{image},
$created,
$expired;
$$Options{'no-run'} and next;
system("rm -rf $$expire{path}/tree");
$$Options{tree} and next;
system("rm -rf $$expire{path}");
}
exit 0;
sub check_expire
{
my ($summary, $expire_time) = @_;
my ($expire, $etime, $path);
$expire = $$summary{Expire};
$expire =~ s/^.*==\s+//;
$expire or return 0;
$expire =~ /never/i and return 0;
$etime = parsedate($expire);
if (!$etime)
{
print STDERR "$File::Find::dir: invalid expiration time $$summary{expire}\n";
return -1;
}
$etime > $expire_time and return 0;
return 1;
}
sub findop
{
if ($_ eq 'tree')
{
$File::Find::prune = 1;
return 0;
}
if ($_ eq 'summary')
{
my $summary;
my ($etime, $path);
$path = $File::Find::dir;
$summary = loadconfig('R', $File::Find::name);
$status = check_expire($summary, $expire_time);
$status < 0 and return;
$$summary{vault} && $$summary{branch} && $$summary{Image}
or return;
if ($status == 0)
{
$$summary{Status} =~ /^success/ && -d ($path . '/tree')
and ++$unexpired{$$summary{vault}}{$$summary{branch}};
return;
}
-d ($path . ($$Options{tree} ? '/tree': undef)) or return;
push (@expires, {
vault => $$summary{vault},
branch => $$summary{branch},
client => $$summary{client},
tree => $$summary{tree},
image => $$summary{Image},
created => $$summary{'Backup-complete'},
expire => $$summary{Expire},
status => $$summary{Status},
path => $path,
}
);
}
}
## WARNING: don't mess with the sort order, it is needed so that if
## WARNING: all images are expired the newest will be retained.
sub imsort
{
$$a{vault} cmp $$b{vault}
|| $$a{branch} cmp $$b{branch}
|| $$a{created} cmp $$b{created};
}
# Get patch level of loadconfig.pl in case exit codes
# are needed.
# $Id: loadconfig.pl,v 12.0 2004/02/25 02:42:15 jw Exp $
#########################################################################
# #
# Copyright 2002 and $Date: 2004/02/25 02:42:15 $
# Pegasystems Technologies and J.W. Schultz #
# #
# Licensed under the Open Software License version 2.0 #
# #
#########################################################################
sub seppuku # Exit with code and message.
{
my ($status, $message) = @_;
chomp $message;
if ($message)
{
$seppuku_prefix and print STDERR $seppuku_prefix, ': ';
print STDERR $message, "\n";
}
exit $status;
}
sub slurplist
{
my ($key, $filename, $Options) = @_;
my $f;
my $array;
$filename =~ m(^/) and $f = $filename;
if (!$f && ref($$Options{vault}) ne 'CODE')
{
$f = join('/', $$Options{Bank}, $$Options{vault},
'dirvish', $filename);
-f $f or $f = undef;
}
$f or $f = "$CONFDIR/$filename";
open(PATFILE, "<$f") or seppuku 229, "cannot open $filename for $key list";
$array = $$Options{$key};
while(<PATFILE>)
{
chomp;
length or next;
push @{$array}, $_;
}
close PATFILE;
}
# loadconfig -- load configuration file
# SYNOPSYS
# loadconfig($opts, $filename, \%data)
#
# DESCRIPTION
# load and parse a configuration file into the data
# hash. If the filename does not contain / it will be
# looked for in the vault if defined. If the filename
# does not exist but filename.conf does that will
# be read.
#
# OPTIONS
# Options are case sensitive, upper case has the
# opposite effect of lower case. If conflicting
# options are given only the last will have effect.
#
# f Ignore fields in config file that are
# capitalized.
#
# o Config file is optional, return undef if missing.
#
# R Do not allow recoursion.
#
# g Only load from global directory.
#
#
#
# LIMITATIONS
# Only way to tell whether an option should be a list
# or scalar is by the formatting in the config file.
#
# Options reqiring special handling have to have that
# hardcoded in the function.
#
sub loadconfig
{
my ($mode, $configfile, $Options) = @_;
my $confile = undef;
my ($key, $val);
my $CONFIG;
ref($Options) or $Options = {};
my %modes;
my ($conf, $bank, $k);
$modes{r} = 1;
for $_ (split(//, $mode))
{
if (/[A-Z]/)
{
$_ =~ tr/A-Z/a-z/;
$modes{$_} = 0;
} else {
$modes{$_} = 1;
}
}
$CONFIG = 'CFILE' . scalar(@{$$Options{Configfiles}});
$configfile =~ s/^.*\@//;
if($configfile =~ m[/])
{
$confile = $configfile;
}
elsif($configfile ne '-')
{
if(!$modes{g} && $$Options{vault} && $$Options{vault} ne 'CODE')
{
if(!$$Options{Bank})
{
my $bank;
for $bank (@{$$Options{bank}})
{
if (-d "$bank/$$Options{vault}")
{
$$Options{Bank} = $bank;
last;
}
}
}
if ($$Options{Bank})
{
$confile = join('/', $$Options{Bank},
$$Options{vault}, 'dirvish',
$configfile);
-f $confile || -f "$confile.conf"
or $confile = undef;
}
}
$confile ||= "$CONFDIR/$configfile";
}
if($configfile eq '-')
{
open($CONFIG, $configfile) or seppuku 221, "cannot open STDIN";
} else {
! -f $confile && -f "$confile.conf" and $confile .= '.conf';
if (! -f "$confile")
{
$modes{o} and return undef;
seppuku 222, "cannot open config file: $configfile";
}
grep(/^$confile$/, @{$$Options{Configfiles}})
and seppuku 224, "ERROR: config file looping on $confile";
open($CONFIG, $confile)
or seppuku 225, "cannot open config file: $configfile";
}
push(@{$$Options{Configfiles}}, $confile);
while(<$CONFIG>)
{
chomp;
s/\s*#.*$//;
s/\s+$//;
/\S/ or next;
if(/^\s/ && $key)
{
s/^\s*//;
push @{$$Options{$key}}, $_;
}
elsif(/^SET\s+/)
{
s/^SET\s+//;
for $k (split(/\s+/))
{
$$Options{$k} = 1;
}
}
elsif(/^UNSET\s+/)
{
s/^UNSET\s+//;
for $k (split(/\s+/))
{
$$Options{$k} = undef;
}
}
elsif(/^RESET\s+/)
{
($key = $_) =~ s/^RESET\s+//;
$$Options{$key} = [ ];
}
elsif(/^[A-Z]/ && $modes{f})
{
$key = undef;
}
elsif(/^\S+:/)
{
($key, $val) = split(/:\s*/, $_, 2);
length($val) or next;
$k = $key; $key = undef;
if ($k eq 'config')
{
$modes{r} and loadconfig($mode . 'O', $val, $Options);
next;
}
if ($k eq 'client')
{
if ($modes{r} && ref ($$Options{$k}) eq 'CODE')
{
loadconfig($mode . 'og', "$CONFDIR/$val", $Options);
}
$$Options{$k} = $val;
next;
}
if ($k eq 'file-exclude')
{
$modes{r} or next;
slurplist('exclude', $val, $Options);
next;
}
if (ref ($$Options{$k}) eq 'ARRAY')
{
push @{$$Options{$k}}, $_;
} else {
$$Options{$k} = $val;
}
}
}
close $CONFIG;
return $Options;
}

View file

@ -0,0 +1,16 @@
{{ ansible_managed | comment }}
SHELL=/bin/sh
{%- set hosts = [] -%}
{%- for host in hostvars.keys() -%}
{%- if dirvish_server_clients_group in hostvars[host].group_names -%}
{%- set _ = hosts.append(host) -%}
{%- endif -%}
{%- endfor %}
{% for host in hosts %}
{%- set t = dirvish_server_cronjob_minute_initial + dirvish_server_cronjob_step_minutes * loop.index0 -%}
{%- set m = t % 60 -%}
{%- set h = ( dirvish_server_cronjob_hour_initial + ((t/60)|int) ) % 24 -%}
{{ '% 2s'|format(m) }} {{ '% 2s'|format(h) }} {{ '% 2s'|format(dirvish_server_cronjob_dom) }} {{ '% 2s'|format(dirvish_server_cronjob_month) }} {{ '% 2s'|format(dirvish_server_cronjob_dow) }} root /usr/local/bin/dirvish-expire-configfile --config {{ dirvish_server_master_conf }} --vault {{ host }} --quiet && /usr/sbin/dirvish --config {{ dirvish_server_master_conf }} --vault {{ host }}
{% endfor %}

View file

@ -0,0 +1,25 @@
{{ ansible_managed | comment }}
bank:
{% set banks = [] -%}
{%- for host in hostvars.keys() -%}
{%- if dirvish_server_clients_group in hostvars[host].group_names -%}
{%- set _ = banks.append(hostvars[host].dirvish_server_bank) -%}
{%- endif -%}
{%- endfor -%}
{% for bank in (banks | unique) %}
{{ bank }}
{% endfor %}
branch-default: {{ dirvish_server_branch_default }}
image-default: {{ dirvish_server_image_name }}
log: {{ dirvish_server_log_compression }}
index: {{ dirvish_server_log_compression }}
expire-default: {{ dirvish_server_expire_default }}
{% if dirvish_server_exclude|length > 0 %}
exclude:
{% for exclude in dirvish_server_exclude %}
{{ exclude }}
{% endfor %}
{% endif %}