verdnatura/vn-vmware
verdnatura
/
vn-vmware
Archived
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactor, backup replication ready
gitea/vn-vmware/pipeline/head There was a failure building this commit Details

This commit is contained in:
Juan Ferrer 2020-05-30 19:51:25 +02:00
parent db07203605
commit c8e5fa806a
7 changed files with 218 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
config.my.pl
config.local.pl
visdkrc

11
config.pl
View File

@ -1,5 +1,5 @@
# Configuration file, be careful to respect the Perl syntax.
# Don't modify this file, copy it to config.my.pl and make your changes there.
# Don't modify this file, copy it to config.local.pl and make your changes there.
# The log file path
log_file => '/var/log/vn-vmware.log',
@ -10,18 +10,21 @@ datacenter => 'datacenter1',
# The datastore name where backups must be stored
backup_datastore => 'backup',
# The local directory where backups datastore folder is mounted
# Directory where backups are stored
# If it's mounted via NFS you have to disable caching for the mount
local_backup_dir => '/mnt/backup',
# Directory where backups will be restored
# Directory where backups are be restored
restore_dir => '/mnt/backup',
# Directory where local backup directory is replicated
replicate_dir => '/mnt/backup-sync',
# Number of processes used by pigz to compress backups
pigz_processes => 1,
# Whether to encrypt backups using gpg with passed passphrase file
passphrase_file => '/etc/vn-vmware/passphrase.gpg',
passphrase_file => '/etc/vn-vmware/gpg.key',
# The default rotation to use if none is specified on job
rotation => 'lastMonth',

2
debian/changelog vendored
View File

@ -1,4 +1,4 @@
vn-vmware (1.1.21) stable; urgency=low
vn-vmware (1.1.22) stable; urgency=low
* Initial Release.

2
debian/control vendored
View File

@ -10,7 +10,7 @@ Vcs-Git: https://git.verdnatura.es/vn-vmware
Package: vn-vmware
Architecture: all
Depends: perl, pigz, libsys-cpu-perl
Suggests: gpg
Suggests: gpg, rsync
Section: misc
Priority: optional
Description: Maintenance scripts for VMWare

2
debian/install vendored
View File

@ -1,2 +1,2 @@
config.pl etc/vn-vmware
vn-vmware.pl usr/share/vn-vmware
vn-vmware.pl usr/share/vn-vmware

5
deploy
View File

@ -1,5 +0,0 @@
#!/bin/bash
set -e
vn-debuild
vn-deploy root@bacula3

330
vn-vmware.pl
View File

@ -24,9 +24,9 @@ use constant true => 1;
my %config;
my @config_files = (
'/etc/vn-vmware/config.my.pl',
'/etc/vn-vmware/config.local.pl',
'/etc/vn-vmware/config.pl',
'config.my.pl',
'config.local.pl',
'config.pl'
);
@ -56,7 +56,7 @@ foreach my $vi_config_file (@vi_config_files) {
my %opts = (
'operation' => {
type => "=s",
help => "Operation to perform: none, backup-job, clone-job, backup, rotate, restore, clone, snapshot, migrate",
help => "Operation to perform: none, backup, backup-job, rotate, restore, replicate, init, clone, clone-job, snapshot, migrate",
required => true
},
'job' => {
@ -92,9 +92,13 @@ my %opts = (
},
'restore-dir' => {
type => "=s",
help => "The backup directory where backup will be restored",
help => "Directory where backups are be restored",
default => '.'
},
'replicate-dir' => {
type => "=s",
help => "Directory where local backup directory is replicated"
},
'dst-name' => {
type => "=s",
help => "Name of the new virtual machine"
@ -171,6 +175,21 @@ Opts::add_options(%opts);
Opts::parse();
Opts::validate();
my @config_options = (
'passphrase-file',
'restore-dir',
'replicate-dir'
);
foreach my $config_option (@config_options) {
my $underscore = $config_option;
$underscore =~ s/-/_/;
if (!Opts::option_is_set($config_option) && exists $config{$underscore}) {
Opts::set_option($config_option, $config{$underscore});
}
}
Opts::validate();
my $server = Opts::get_option('server');
my $operation = Opts::get_option('operation');
my $job = Opts::get_option('job');
@ -181,6 +200,7 @@ my $rotation_count = Opts::get_option('rotation-count');
my $archive_regex = Opts::get_option('archive-regex');
my $backup_file = Opts::get_option('backup-file');
my $restore_dir = Opts::get_option('restore-dir');
my $replicate_dir = Opts::get_option('replicate-dir');
my $dst_name = Opts::get_option('dst-name');
my $dst_host = Opts::get_option('dst-host');
my $dst_datastore = Opts::get_option('dst-datastore');
@ -197,16 +217,13 @@ my $test = Opts::option_is_set('test');
my $snapshot_name = Opts::get_option('snapshot-name');
my $snapshot_desc = Opts::get_option('snapshot-desc');
if (!defined ($passphrase_file) && exists $config{passphrase_file}) {
$passphrase_file = $config{passphrase_file};
}
my $vm;
my $log_fh;
my $archive_fn;
my $backup_disks;
my $time_pattern = '%Y-%m-%d_%H-%M';
my $local_backup_dir = $config{local_backup_dir};
my $secure_file = "$local_backup_dir/.keepme";
sub stringify_message {
my ($message) = @_;
@ -288,26 +305,32 @@ sub main {
die "Operation not defined.";
}
given ($operation) {
when ('backup-job') {
backup_job();
}
when ('clone-job') {
clone_job();
}
when ('backup') {
open_machine();
backup_machine();
}
when ('backup-job') {
backup_job();
}
when ('rotate') {
rotate_backup();
}
when ('restore') {
restore_backup();
}
when ('replicate') {
replicate_backups();
}
when ('init') {
init_backup_dir();
}
when ('clone') {
open_machine();
clone_machine();
}
when ('clone-job') {
clone_job();
}
when ('snapshot') {
open_machine();
snapshot_machine();
@ -336,119 +359,30 @@ sub main {
#--------------------------------- Operations
sub backup_job() {
unless ($job) {
die "Job not defined.";
}
unless (exists $config{backup_jobs}{$job}) {
die "Backup job '$job' doesn't exist.";
}
log_message "Backup job '$job' started.";
my $backup_job = $config{backup_jobs}{$job};
my @machines = @{$backup_job->{machines}};
my $default_rotation;
if (exists $backup_job->{rotation}) {
$default_rotation = $backup_job->{rotation};
} elsif (exists $config{rotation}) {
$default_rotation = $config{rotation};
}
foreach my $machine (@machines) {
eval {
my $rotation_name = $default_rotation;
if (ref($machine) eq 'HASH') {
$vm_name = $machine->{name};
$backup_disks = $machine->{disks};
if (exists $machine->{rotation}) {
$rotation_name = $machine->{rotation};
}
} else {
$vm_name = $machine;
}
if (defined ($rotation_name)) {
unless (exists $config{rotations}{$rotation_name}) {
die "Rotation '$rotation_name' not defined.";
}
my $rotation_cfg = $config{rotations}{$rotation_name};
if (exists $rotation_cfg->{count}) {
$rotation_count = $rotation_cfg->{count};
}
if (exists $rotation_cfg->{days}) {
$rotation_days = $rotation_cfg->{days};
}
if (exists $rotation_cfg->{archive_regex}) {
$archive_regex = $rotation_cfg->{archive_regex};
$archive_regex = qr/$archive_regex/;
}
if (exists $rotation_cfg->{archive_fn}) {
$archive_fn = $rotation_cfg->{archive_fn};
}
}
open_machine();
backup_machine();
rotate_backup();
};
if ($@) {
log_error $@;
}
}
log_message "Backup job '$job' finished.";
}
sub clone_job() {
unless ($job) {
die "Job not defined.";
}
unless (exists $config{clone_jobs}{$job}) {
die "Clone job '$job' doesn't exist.";
}
log_message "Clone job '$job' started.";
my $clone_job = $config{clone_jobs}{$job};
$vm_name = $clone_job->{vm};
$dst_name = $clone_job->{dst_name};
$dst_host = $clone_job->{dst_host};
$dst_datastore = $clone_job->{dst_datastore};
$memory = $clone_job->{memory};
$num_cpus = $clone_job->{num_cpus};
$mac = $clone_job->{mac};
$poweron = $clone_job->{poweron};
$overwrite = $clone_job->{overwrite};
open_machine();
clone_machine();
log_message "Clone job '$job' finished.";
}
sub backup_machine() {
log_message "Backup of '$vm_name' started.";
my $time = Time::Piece->new;
my $time_mark = $time->strftime($time_pattern);
my $tmpDir = ".$time_mark";
my $tmp_dir = ".$time_mark";
my $backup_datastore = $config{backup_datastore};
my $ds_tmp_dir = "[$backup_datastore] $vm_name/$tmpDir";
my $ds_tmp_dir = "[$backup_datastore] $vm_name/$tmp_dir";
my $local_dir = "$local_backup_dir/$vm_name";
my $local_tmp_dir = "$local_dir/$tmpDir";
my $local_tmp_dir = "$local_dir/$tmp_dir";
my $tar_file = "$local_dir/${vm_name}_$time_mark.tar.gz";
if (defined($passphrase_file)) {
$tar_file = "$tar_file.gpg";
}
my $tmp_file = "$tar_file.tmp";
if (-e $local_tmp_dir) {
die "Temporary backup directory already exists: $local_tmp_dir";
}
if (-e $tar_file) {
die "Backup file already exists: $tar_file";
}
my $service_content = Vim::get_service_content();
my $file_manager = Vim::get_view(mo_ref => $service_content->fileManager);
@ -589,13 +523,13 @@ sub backup_machine() {
}
log_message "Removing backup snapshot.";
unless ($test) {
if ($snapshot) {
$snapshot->RemoveSnapshot(
removeChildren => true,
consolidate => true
);
$snapshot = undef;
}
$snapshot = undef;
my $pigz_processes;
@ -609,10 +543,10 @@ sub backup_machine() {
my $tar_command = "tar -I \"pigz -p $pigz_processes\" -c -C \"$local_tmp_dir\" .";
if (defined($passphrase_file)) {
my $gpg_command = "gpg -c --passphrase-file \"$passphrase_file\" --batch --yes -o \"$tar_file.gpg\"";
my $gpg_command = "gpg -c --passphrase-file \"$passphrase_file\" --batch --yes -o \"$tmp_file\"";
$tar_command = "$tar_command | $gpg_command";
} else {
$tar_command = "$tar_command -f \"$tar_file\"";
$tar_command = "$tar_command -f \"$tmp_file\"";
}
log_message "Compressing with Gzip (using $pigz_processes processes) to TAR file.";
@ -629,6 +563,10 @@ sub backup_machine() {
}
setpriority(0, 0, $priority);
unless ($test) {
move($tmp_file, $tar_file);
}
};
my $err = $@;
@ -642,6 +580,14 @@ sub backup_machine() {
consolidate => true
);
}
unless ($test) {
if (-e $tar_file) {
unlink $tar_file;
}
if (-e $tmp_file) {
unlink $tmp_file;
}
}
}
log_message "Removing temporary directory: $local_tmp_dir";
@ -656,6 +602,76 @@ sub backup_machine() {
log_message "Backup of '$vm_name' successfully created.";
}
sub backup_job() {
unless ($job) {
die "Job not defined.";
}
unless (exists $config{backup_jobs}{$job}) {
die "Backup job '$job' doesn't exist.";
}
log_message "Backup job '$job' started.";
my $backup_job = $config{backup_jobs}{$job};
my @machines = @{$backup_job->{machines}};
my $default_rotation;
if (exists $backup_job->{rotation}) {
$default_rotation = $backup_job->{rotation};
} elsif (exists $config{rotation}) {
$default_rotation = $config{rotation};
}
foreach my $machine (@machines) {
eval {
my $rotation_name = $default_rotation;
if (ref($machine) eq 'HASH') {
$vm_name = $machine->{name};
$backup_disks = $machine->{disks};
if (exists $machine->{rotation}) {
$rotation_name = $machine->{rotation};
}
} else {
$vm_name = $machine;
}
if (defined ($rotation_name)) {
unless (exists $config{rotations}{$rotation_name}) {
die "Rotation '$rotation_name' not defined.";
}
my $rotation_cfg = $config{rotations}{$rotation_name};
if (exists $rotation_cfg->{count}) {
$rotation_count = $rotation_cfg->{count};
}
if (exists $rotation_cfg->{days}) {
$rotation_days = $rotation_cfg->{days};
}
if (exists $rotation_cfg->{archive_regex}) {
$archive_regex = $rotation_cfg->{archive_regex};
$archive_regex = qr/$archive_regex/;
}
if (exists $rotation_cfg->{archive_fn}) {
$archive_fn = $rotation_cfg->{archive_fn};
}
}
open_machine();
backup_machine();
rotate_backup();
};
if ($@) {
log_error $@;
}
}
log_message "Backup job '$job' finished.";
}
sub rotate_backup() {
if ($rotation_days == 0 && $rotation_count == 0) {
log_message "No rotation days or count, aborting.";
@ -717,7 +733,7 @@ sub rotate_backup() {
foreach my $archive_file (@archive_files) {
log_message "$archive_file";
unless ($test) {
move ("$local_dir/$archive_file", "$archive_dir/$archive_file");
move("$local_dir/$archive_file", "$archive_dir/$archive_file");
}
}
@ -764,9 +780,6 @@ sub restore_backup() {
die "Backup file doesn't exists: $backup_file"
}
if (!defined ($restore_dir) && exists $config{restore_dir}) {
$restore_dir = $config{restore_dir};
}
unless (defined($restore_dir)) {
die "Restore directory not defined."
}
@ -794,6 +807,49 @@ sub restore_backup() {
log_message "Backup restored successfully.";
}
sub replicate_backups() {
log_message "Replicating backups to '$replicate_dir'.";
unless (-e $secure_file) {
die "Invalid source directory. Is it mounted and initialized?";
}
unless (-e $replicate_dir) {
die "Replicate dir doesn't exists: $replicate_dir";
}
my $rsync_params = '-rltmD --delete-after --include="*.tar.gz" --include="*.tar.gz.gpg" --include="*/" --exclude="*"';
my $rsync_command = "rsync $rsync_params \"$local_backup_dir/\" \"$replicate_dir\"";
log_message $rsync_command;
unless ($test) {
my $tar_status = system($rsync_command);
unless ($rsync_command == 0) {
die "An error occurred while replicating backups.";
}
}
log_message "Backups replicated successfully.";
}
sub init_backup_dir() {
log_message "Initializing backup directory '$local_backup_dir'.";
if (-e $secure_file) {
die "Backup directory already initialized.";
}
my $secure_content = "Don't delete me! I'm used to be sure this directory is mounted before running rsync\n";
unless ($test) {
my $secure_fh;
open($secure_fh, '>>', $secure_file);
print $secure_fh $secure_content;
close($secure_fh);
}
log_message "Backup directory initalized.";
}
sub clone_machine {
log_message "Cloning '$vm_name' to '$dst_name'.";
log_message "Doing some previous checkings.";
@ -992,6 +1048,34 @@ sub clone_machine {
log_message "Clone '$dst_name' of '$vm_name' successfully created.";
}
sub clone_job() {
unless ($job) {
die "Job not defined.";
}
unless (exists $config{clone_jobs}{$job}) {
die "Clone job '$job' doesn't exist.";
}
log_message "Clone job '$job' started.";
my $clone_job = $config{clone_jobs}{$job};
$vm_name = $clone_job->{vm};
$dst_name = $clone_job->{dst_name};
$dst_host = $clone_job->{dst_host};
$dst_datastore = $clone_job->{dst_datastore};
$memory = $clone_job->{memory};
$num_cpus = $clone_job->{num_cpus};
$mac = $clone_job->{mac};
$poweron = $clone_job->{poweron};
$overwrite = $clone_job->{overwrite};
open_machine();
clone_machine();
log_message "Clone job '$job' finished.";
}
sub snapshot_machine() {
log_message "Creating snapshot of '$vm_name'.";