diff --git a/.gitignore b/.gitignore index 2579bda..5a5db7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -config.my.pl +config.local.pl visdkrc \ No newline at end of file diff --git a/config.pl b/config.pl index 0aca690..28b40c3 100644 --- a/config.pl +++ b/config.pl @@ -1,5 +1,5 @@ # Configuration file, be careful to respect the Perl syntax. -# Don't modify this file, copy it to config.my.pl and make your changes there. +# Don't modify this file, copy it to config.local.pl and make your changes there. # The log file path log_file => '/var/log/vn-vmware.log', @@ -10,18 +10,21 @@ datacenter => 'datacenter1', # The datastore name where backups must be stored backup_datastore => 'backup', -# The local directory where backups datastore folder is mounted +# Directory where backups are stored # If it's mounted via NFS you have to disable caching for the mount local_backup_dir => '/mnt/backup', -# Directory where backups will be restored +# Directory where backups are be restored restore_dir => '/mnt/backup', +# Directory where local backup directory is replicated +replicate_dir => '/mnt/backup-sync', + # Number of processes used by pigz to compress backups pigz_processes => 1, # Whether to encrypt backups using gpg with passed passphrase file -passphrase_file => '/etc/vn-vmware/passphrase.gpg', +passphrase_file => '/etc/vn-vmware/gpg.key', # The default rotation to use if none is specified on job rotation => 'lastMonth', diff --git a/debian/changelog b/debian/changelog index 4de8bdf..05eaffe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -vn-vmware (1.1.21) stable; urgency=low +vn-vmware (1.1.22) stable; urgency=low * Initial Release. diff --git a/debian/control b/debian/control index 9b8777f..c22e8ac 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Vcs-Git: https://git.verdnatura.es/vn-vmware Package: vn-vmware Architecture: all Depends: perl, pigz, libsys-cpu-perl -Suggests: gpg +Suggests: gpg, rsync Section: misc Priority: optional Description: Maintenance scripts for VMWare diff --git a/debian/install b/debian/install index b8bf86d..4f11074 100644 --- a/debian/install +++ b/debian/install @@ -1,2 +1,2 @@ config.pl etc/vn-vmware -vn-vmware.pl usr/share/vn-vmware \ No newline at end of file +vn-vmware.pl usr/share/vn-vmware diff --git a/deploy b/deploy deleted file mode 100755 index b2fb642..0000000 --- a/deploy +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e - -vn-debuild -vn-deploy root@bacula3 diff --git a/vn-vmware.pl b/vn-vmware.pl index f673224..7bdc526 100755 --- a/vn-vmware.pl +++ b/vn-vmware.pl @@ -24,9 +24,9 @@ use constant true => 1; my %config; my @config_files = ( - '/etc/vn-vmware/config.my.pl', + '/etc/vn-vmware/config.local.pl', '/etc/vn-vmware/config.pl', - 'config.my.pl', + 'config.local.pl', 'config.pl' ); @@ -56,7 +56,7 @@ foreach my $vi_config_file (@vi_config_files) { my %opts = ( 'operation' => { type => "=s", - help => "Operation to perform: none, backup-job, clone-job, backup, rotate, restore, clone, snapshot, migrate", + help => "Operation to perform: none, backup, backup-job, rotate, restore, replicate, init, clone, clone-job, snapshot, migrate", required => true }, 'job' => { @@ -92,9 +92,13 @@ my %opts = ( }, 'restore-dir' => { type => "=s", - help => "The backup directory where backup will be restored", + help => "Directory where backups are be restored", default => '.' }, + 'replicate-dir' => { + type => "=s", + help => "Directory where local backup directory is replicated" + }, 'dst-name' => { type => "=s", help => "Name of the new virtual machine" @@ -171,6 +175,21 @@ Opts::add_options(%opts); Opts::parse(); Opts::validate(); +my @config_options = ( + 'passphrase-file', + 'restore-dir', + 'replicate-dir' +); +foreach my $config_option (@config_options) { + my $underscore = $config_option; + $underscore =~ s/-/_/; + if (!Opts::option_is_set($config_option) && exists $config{$underscore}) { + Opts::set_option($config_option, $config{$underscore}); + } +} + +Opts::validate(); + my $server = Opts::get_option('server'); my $operation = Opts::get_option('operation'); my $job = Opts::get_option('job'); @@ -181,6 +200,7 @@ my $rotation_count = Opts::get_option('rotation-count'); my $archive_regex = Opts::get_option('archive-regex'); my $backup_file = Opts::get_option('backup-file'); my $restore_dir = Opts::get_option('restore-dir'); +my $replicate_dir = Opts::get_option('replicate-dir'); my $dst_name = Opts::get_option('dst-name'); my $dst_host = Opts::get_option('dst-host'); my $dst_datastore = Opts::get_option('dst-datastore'); @@ -197,16 +217,13 @@ my $test = Opts::option_is_set('test'); my $snapshot_name = Opts::get_option('snapshot-name'); my $snapshot_desc = Opts::get_option('snapshot-desc'); -if (!defined ($passphrase_file) && exists $config{passphrase_file}) { - $passphrase_file = $config{passphrase_file}; -} - my $vm; my $log_fh; my $archive_fn; my $backup_disks; my $time_pattern = '%Y-%m-%d_%H-%M'; my $local_backup_dir = $config{local_backup_dir}; +my $secure_file = "$local_backup_dir/.keepme"; sub stringify_message { my ($message) = @_; @@ -288,26 +305,32 @@ sub main { die "Operation not defined."; } given ($operation) { - when ('backup-job') { - backup_job(); - } - when ('clone-job') { - clone_job(); - } when ('backup') { open_machine(); backup_machine(); } + when ('backup-job') { + backup_job(); + } when ('rotate') { rotate_backup(); } when ('restore') { restore_backup(); } + when ('replicate') { + replicate_backups(); + } + when ('init') { + init_backup_dir(); + } when ('clone') { open_machine(); clone_machine(); } + when ('clone-job') { + clone_job(); + } when ('snapshot') { open_machine(); snapshot_machine(); @@ -336,119 +359,30 @@ sub main { #--------------------------------- Operations -sub backup_job() { - unless ($job) { - die "Job not defined."; - } - unless (exists $config{backup_jobs}{$job}) { - die "Backup job '$job' doesn't exist."; - } - - log_message "Backup job '$job' started."; - - my $backup_job = $config{backup_jobs}{$job}; - my @machines = @{$backup_job->{machines}}; - - my $default_rotation; - - if (exists $backup_job->{rotation}) { - $default_rotation = $backup_job->{rotation}; - } elsif (exists $config{rotation}) { - $default_rotation = $config{rotation}; - } - - foreach my $machine (@machines) { - eval { - my $rotation_name = $default_rotation; - - if (ref($machine) eq 'HASH') { - $vm_name = $machine->{name}; - $backup_disks = $machine->{disks}; - - if (exists $machine->{rotation}) { - $rotation_name = $machine->{rotation}; - } - } else { - $vm_name = $machine; - } - - if (defined ($rotation_name)) { - unless (exists $config{rotations}{$rotation_name}) { - die "Rotation '$rotation_name' not defined."; - } - - my $rotation_cfg = $config{rotations}{$rotation_name}; - - if (exists $rotation_cfg->{count}) { - $rotation_count = $rotation_cfg->{count}; - } - if (exists $rotation_cfg->{days}) { - $rotation_days = $rotation_cfg->{days}; - } - if (exists $rotation_cfg->{archive_regex}) { - $archive_regex = $rotation_cfg->{archive_regex}; - $archive_regex = qr/$archive_regex/; - } - if (exists $rotation_cfg->{archive_fn}) { - $archive_fn = $rotation_cfg->{archive_fn}; - } - } - - open_machine(); - backup_machine(); - rotate_backup(); - }; - if ($@) { - log_error $@; - } - } - - log_message "Backup job '$job' finished."; -} - -sub clone_job() { - unless ($job) { - die "Job not defined."; - } - unless (exists $config{clone_jobs}{$job}) { - die "Clone job '$job' doesn't exist."; - } - - log_message "Clone job '$job' started."; - - my $clone_job = $config{clone_jobs}{$job}; - - $vm_name = $clone_job->{vm}; - $dst_name = $clone_job->{dst_name}; - $dst_host = $clone_job->{dst_host}; - $dst_datastore = $clone_job->{dst_datastore}; - $memory = $clone_job->{memory}; - $num_cpus = $clone_job->{num_cpus}; - $mac = $clone_job->{mac}; - $poweron = $clone_job->{poweron}; - $overwrite = $clone_job->{overwrite}; - - open_machine(); - clone_machine(); - - log_message "Clone job '$job' finished."; -} - sub backup_machine() { log_message "Backup of '$vm_name' started."; my $time = Time::Piece->new; my $time_mark = $time->strftime($time_pattern); - my $tmpDir = ".$time_mark"; + my $tmp_dir = ".$time_mark"; my $backup_datastore = $config{backup_datastore}; - my $ds_tmp_dir = "[$backup_datastore] $vm_name/$tmpDir"; + my $ds_tmp_dir = "[$backup_datastore] $vm_name/$tmp_dir"; my $local_dir = "$local_backup_dir/$vm_name"; - my $local_tmp_dir = "$local_dir/$tmpDir"; + my $local_tmp_dir = "$local_dir/$tmp_dir"; my $tar_file = "$local_dir/${vm_name}_$time_mark.tar.gz"; + if (defined($passphrase_file)) { + $tar_file = "$tar_file.gpg"; + } + + my $tmp_file = "$tar_file.tmp"; + if (-e $local_tmp_dir) { die "Temporary backup directory already exists: $local_tmp_dir"; } + if (-e $tar_file) { + die "Backup file already exists: $tar_file"; + } my $service_content = Vim::get_service_content(); my $file_manager = Vim::get_view(mo_ref => $service_content->fileManager); @@ -589,13 +523,13 @@ sub backup_machine() { } log_message "Removing backup snapshot."; - unless ($test) { + if ($snapshot) { $snapshot->RemoveSnapshot( removeChildren => true, consolidate => true ); + $snapshot = undef; } - $snapshot = undef; my $pigz_processes; @@ -609,10 +543,10 @@ sub backup_machine() { my $tar_command = "tar -I \"pigz -p $pigz_processes\" -c -C \"$local_tmp_dir\" ."; if (defined($passphrase_file)) { - my $gpg_command = "gpg -c --passphrase-file \"$passphrase_file\" --batch --yes -o \"$tar_file.gpg\""; + my $gpg_command = "gpg -c --passphrase-file \"$passphrase_file\" --batch --yes -o \"$tmp_file\""; $tar_command = "$tar_command | $gpg_command"; } else { - $tar_command = "$tar_command -f \"$tar_file\""; + $tar_command = "$tar_command -f \"$tmp_file\""; } log_message "Compressing with Gzip (using $pigz_processes processes) to TAR file."; @@ -629,6 +563,10 @@ sub backup_machine() { } setpriority(0, 0, $priority); + unless ($test) { + move($tmp_file, $tar_file); + } + }; my $err = $@; @@ -642,6 +580,14 @@ sub backup_machine() { consolidate => true ); } + unless ($test) { + if (-e $tar_file) { + unlink $tar_file; + } + if (-e $tmp_file) { + unlink $tmp_file; + } + } } log_message "Removing temporary directory: $local_tmp_dir"; @@ -656,6 +602,76 @@ sub backup_machine() { log_message "Backup of '$vm_name' successfully created."; } +sub backup_job() { + unless ($job) { + die "Job not defined."; + } + unless (exists $config{backup_jobs}{$job}) { + die "Backup job '$job' doesn't exist."; + } + + log_message "Backup job '$job' started."; + + my $backup_job = $config{backup_jobs}{$job}; + my @machines = @{$backup_job->{machines}}; + + my $default_rotation; + + if (exists $backup_job->{rotation}) { + $default_rotation = $backup_job->{rotation}; + } elsif (exists $config{rotation}) { + $default_rotation = $config{rotation}; + } + + foreach my $machine (@machines) { + eval { + my $rotation_name = $default_rotation; + + if (ref($machine) eq 'HASH') { + $vm_name = $machine->{name}; + $backup_disks = $machine->{disks}; + + if (exists $machine->{rotation}) { + $rotation_name = $machine->{rotation}; + } + } else { + $vm_name = $machine; + } + + if (defined ($rotation_name)) { + unless (exists $config{rotations}{$rotation_name}) { + die "Rotation '$rotation_name' not defined."; + } + + my $rotation_cfg = $config{rotations}{$rotation_name}; + + if (exists $rotation_cfg->{count}) { + $rotation_count = $rotation_cfg->{count}; + } + if (exists $rotation_cfg->{days}) { + $rotation_days = $rotation_cfg->{days}; + } + if (exists $rotation_cfg->{archive_regex}) { + $archive_regex = $rotation_cfg->{archive_regex}; + $archive_regex = qr/$archive_regex/; + } + if (exists $rotation_cfg->{archive_fn}) { + $archive_fn = $rotation_cfg->{archive_fn}; + } + } + + open_machine(); + backup_machine(); + rotate_backup(); + }; + if ($@) { + log_error $@; + } + } + + log_message "Backup job '$job' finished."; +} + sub rotate_backup() { if ($rotation_days == 0 && $rotation_count == 0) { log_message "No rotation days or count, aborting."; @@ -717,7 +733,7 @@ sub rotate_backup() { foreach my $archive_file (@archive_files) { log_message "$archive_file"; unless ($test) { - move ("$local_dir/$archive_file", "$archive_dir/$archive_file"); + move("$local_dir/$archive_file", "$archive_dir/$archive_file"); } } @@ -764,9 +780,6 @@ sub restore_backup() { die "Backup file doesn't exists: $backup_file" } - if (!defined ($restore_dir) && exists $config{restore_dir}) { - $restore_dir = $config{restore_dir}; - } unless (defined($restore_dir)) { die "Restore directory not defined." } @@ -794,6 +807,49 @@ sub restore_backup() { log_message "Backup restored successfully."; } +sub replicate_backups() { + log_message "Replicating backups to '$replicate_dir'."; + + unless (-e $secure_file) { + die "Invalid source directory. Is it mounted and initialized?"; + } + unless (-e $replicate_dir) { + die "Replicate dir doesn't exists: $replicate_dir"; + } + + my $rsync_params = '-rltmD --delete-after --include="*.tar.gz" --include="*.tar.gz.gpg" --include="*/" --exclude="*"'; + my $rsync_command = "rsync $rsync_params \"$local_backup_dir/\" \"$replicate_dir\""; + log_message $rsync_command; + + unless ($test) { + my $tar_status = system($rsync_command); + unless ($rsync_command == 0) { + die "An error occurred while replicating backups."; + } + } + + log_message "Backups replicated successfully."; +} + +sub init_backup_dir() { + log_message "Initializing backup directory '$local_backup_dir'."; + + if (-e $secure_file) { + die "Backup directory already initialized."; + } + + my $secure_content = "Don't delete me! I'm used to be sure this directory is mounted before running rsync\n"; + + unless ($test) { + my $secure_fh; + open($secure_fh, '>>', $secure_file); + print $secure_fh $secure_content; + close($secure_fh); + } + + log_message "Backup directory initalized."; +} + sub clone_machine { log_message "Cloning '$vm_name' to '$dst_name'."; log_message "Doing some previous checkings."; @@ -992,6 +1048,34 @@ sub clone_machine { log_message "Clone '$dst_name' of '$vm_name' successfully created."; } +sub clone_job() { + unless ($job) { + die "Job not defined."; + } + unless (exists $config{clone_jobs}{$job}) { + die "Clone job '$job' doesn't exist."; + } + + log_message "Clone job '$job' started."; + + my $clone_job = $config{clone_jobs}{$job}; + + $vm_name = $clone_job->{vm}; + $dst_name = $clone_job->{dst_name}; + $dst_host = $clone_job->{dst_host}; + $dst_datastore = $clone_job->{dst_datastore}; + $memory = $clone_job->{memory}; + $num_cpus = $clone_job->{num_cpus}; + $mac = $clone_job->{mac}; + $poweron = $clone_job->{poweron}; + $overwrite = $clone_job->{overwrite}; + + open_machine(); + clone_machine(); + + log_message "Clone job '$job' finished."; +} + sub snapshot_machine() { log_message "Creating snapshot of '$vm_name'.";