diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f77798..26d05bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Unreleased + +- Add optional `endpoint` on `Configuration` for S3-compatible storage (e.g. DigitalOcean Spaces). When set, it is passed to `Fog::Storage.new`. Rake tasks also honor `BKP_ENDPOINT`. + ### v0.0.8 - 2023-07-07 - Add support to hooks (methods to run before/after dump and restore) diff --git a/README.md b/README.md index 99949dd..630632d 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,14 @@ PostgresqlBackup.configure do |config| # is set to S3. config.region = '' + # For S3-compatible APIs (DigitalOcean Spaces, MinIO, etc.), + # set the provider's endpoint URL. Leave empty for Amazon S3; + # the default is ''. + # + # Examples: + # config.endpoint = 'https://nyc3.digitaloceanspaces.com' + config.endpoint = '' + # Backup files are created using a pattern made by the current date # and time. If you want to add a sufix to the files, change this # attribute. @@ -122,6 +130,7 @@ However, you can set (or override) a few things when executing the rake: - repository: `BKP_REPOSITORY='File System' bundle exec rake postgresql_backup:dump` - bucket: `BKP_BUCKET='my-bucket' bundle exec rake postgresql_backup:dump` - region: `BKP_REGION='us-east-1' bundle exec rake postgresql_backup:dump` +- endpoint: `BKP_ENDPOINT='https://nyc3.digitaloceanspaces.com' bundle exec rake postgresql_backup:dump` - remote_path: `BKP_REMOTE_PATH='_backups/database' bundle exec rake postgresql_backup:dump` Be aware that, if the gem is configured to use the file system and you force the task to use S3, AWS related attributes must be set, like the access key and the secret key. @@ -132,6 +141,8 @@ You can combine these variables above any way you want: BKP_REPOSITORY='S3' BKP_BUCKET='my-bucket' BKP_REGION='us-east-1' BKP_REMOTE_PATH='_backups/database' bundle exec rake postgresql_backup:dump ``` +For Spaces, R2, or another S3-compatible host, add `BKP_ENDPOINT` (same value you would set as `config.endpoint` in the initializer). + Important note: config/database.yml is used for database configuration, but you may be prompted for the database user's password. @@ -154,7 +165,8 @@ Again, you can use these environment variables: - repository: `BKP_REPOSITORY='File System' bundle exec rake postgresql_backup:restore` - bucket: `BKP_BUCKET='my-bucket' bundle exec rake postgresql_backup:restore` - region: `BKP_REGION='us-east-1' bundle exec rake postgresql_backup:restore` -- remote_path: `BKP_REMOTE_PATH='_backups/database' bundle exec rake postgresql_backup:dump` +- endpoint: `BKP_ENDPOINT='https://nyc3.digitaloceanspaces.com' bundle exec rake postgresql_backup:restore` +- remote_path: `BKP_REMOTE_PATH='_backups/database' bundle exec rake postgresql_backup:restore` Or make any combination you want with them: diff --git a/lib/configuration.rb b/lib/configuration.rb index 28f7a1d..6511ee2 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -5,6 +5,7 @@ class Configuration :aws_secret_access_key, :backup_folder, :bucket, + :endpoint, :file_suffix, :region, :remote_path, @@ -17,19 +18,23 @@ def initialize( aws_secret_access_key: '', backup_folder: 'db/backups', bucket: '', + endpoint: '', file_suffix: '', region: '', remote_path: '_backups/database/', - repository: 'file system' + repository: 'file system', + hooks: nil ) @aws_access_key_id = aws_access_key_id @aws_secret_access_key = aws_secret_access_key @backup_folder = backup_folder @bucket = bucket + @endpoint = endpoint @file_suffix = file_suffix @region = region @remote_path = remote_path @repository = repository + @hooks = Hooks.new(hooks) end def hooks=(hooks) diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake index b80ee15..d74630f 100644 --- a/lib/tasks/backup.rake +++ b/lib/tasks/backup.rake @@ -69,6 +69,7 @@ namespace :postgresql_backup do config.repository = ENV['BKP_REPOSITORY'] if ENV['BKP_REPOSITORY'].present? config.bucket = ENV['BKP_BUCKET'] if ENV['BKP_BUCKET'].present? config.region = ENV['BKP_REGION'] if ENV['BKP_REGION'].present? + config.endpoint = ENV['BKP_ENDPOINT'] if ENV['BKP_ENDPOINT'].present? config.remote_path = ENV['BKP_REMOTE_PATH'] if ENV['BKP_REMOTE_PATH'].present? config end @@ -101,6 +102,7 @@ namespace :postgresql_backup do show_config_for('File suffix', configuration.file_suffix), configuration.s3? ? show_config_for('Bucket', configuration.bucket) : nil, configuration.s3? ? show_config_for('Region', configuration.region) : nil, + configuration.s3? ? show_config_for('Endpoint', configuration.endpoint) : nil, configuration.s3? ? show_config_for('Remote path', configuration.remote_path) : nil ].compact end diff --git a/lib/tools/s3_storage.rb b/lib/tools/s3_storage.rb index 6c14e1d..a0b62bd 100644 --- a/lib/tools/s3_storage.rb +++ b/lib/tools/s3_storage.rb @@ -8,12 +8,7 @@ module Tools class S3Storage def initialize(configuration) @configuration = configuration - @s3 = Fog::Storage.new( - provider: 'AWS', - region: configuration.region, - aws_access_key_id: configuration.aws_access_key_id, - aws_secret_access_key: configuration.aws_secret_access_key - ) + @s3 = Fog::Storage.new(storage_options) end # Send files to S3. @@ -75,6 +70,18 @@ def download(file_name) attr_reader :configuration, :s3 + def storage_options + { + provider: 'AWS', + region: configuration.region, + aws_access_key_id: configuration.aws_access_key_id, + aws_secret_access_key: configuration.aws_secret_access_key + }.tap do |opts| + ep = configuration.endpoint.to_s.strip + opts[:endpoint] = ep unless ep.empty? + end + end + # Force UTF-8 encoding and remove the production environment from # the `ar_internal_metadata` table, unless the current Rails env # is indeed `production`. diff --git a/spec/configuration_spec.rb b/spec/configuration_spec.rb index 563af5a..904fdbe 100644 --- a/spec/configuration_spec.rb +++ b/spec/configuration_spec.rb @@ -12,6 +12,7 @@ it { expect(subject.aws_access_key_id).to be_empty } it { expect(subject.aws_secret_access_key).to be_empty } it { expect(subject.bucket).to be_empty } + it { expect(subject.endpoint).to be_empty } it { expect(subject.region).to be_empty } it { expect(subject.remote_path).to eq('_backups/database/') } it { expect(subject.hooks).to be_a(Hooks) } @@ -28,6 +29,7 @@ aws_access_key_id: 'aws_access_key_id', aws_secret_access_key: 'aws_secret_access_key', bucket: 'bucket', + endpoint: 'https://s3.example.com', region: 'region', remote_path: 'remote_path', hooks: hooks, @@ -40,6 +42,7 @@ it { expect(subject.aws_access_key_id).to eq('aws_access_key_id') } it { expect(subject.aws_secret_access_key).to eq('aws_secret_access_key') } it { expect(subject.bucket).to eq('bucket') } + it { expect(subject.endpoint).to eq('https://s3.example.com') } it { expect(subject.region).to eq('region') } it { expect(subject.remote_path).to eq('remote_path') } it { expect(subject.hooks).to be_a(Hooks) } diff --git a/spec/tools/database_spec.rb b/spec/tools/database_spec.rb index dc659b7..9689f47 100644 --- a/spec/tools/database_spec.rb +++ b/spec/tools/database_spec.rb @@ -35,7 +35,7 @@ let(:debug) { false } it 'executes pg_dump redirecting the output to a dark hole' do - expect(database).to receive(:system).with(cmd, err: File::NULL) + expect(database).to receive(:system).with(cmd, { err: File::NULL }) database.dump(debug: debug) end diff --git a/spec/tools/s3_storage_spec.rb b/spec/tools/s3_storage_spec.rb index 40e643a..07a5579 100644 --- a/spec/tools/s3_storage_spec.rb +++ b/spec/tools/s3_storage_spec.rb @@ -4,15 +4,59 @@ RSpec.describe Tools::S3Storage do let(:remote_path) { 'db/data' } let(:local_path) { 'db/local/backups' } + let(:endpoint) { '' } let(:configuration) do Configuration.new( remote_path: remote_path, backup_folder: local_path, - bucket: 'my-bucket' + bucket: 'my-bucket', + endpoint: endpoint ) end subject(:storage) { described_class.new(configuration) } + describe 'Fog::Storage initialization' do + let(:fog_storage) { double(directories: double) } + + before do + allow(Fog::Storage).to receive(:new).and_return(fog_storage) + end + + let(:base_options) do + { + provider: 'AWS', + region: '', + aws_access_key_id: '', + aws_secret_access_key: '' + } + end + + it 'omits endpoint when configuration endpoint is blank' do + expect(Fog::Storage).to receive(:new).with(base_options) + described_class.new(configuration) + end + + context 'when endpoint is configured' do + let(:endpoint) { 'https://nyc3.digitaloceanspaces.com' } + + it 'passes endpoint to Fog::Storage' do + expect(Fog::Storage).to receive(:new).with( + base_options.merge(endpoint: 'https://nyc3.digitaloceanspaces.com') + ) + described_class.new(configuration) + end + end + + context 'when endpoint is only whitespace' do + let(:endpoint) { " \t " } + + it 'omits endpoint' do + expect(Fog::Storage).to receive(:new).with(base_options) + described_class.new(configuration) + end + end + end + describe '#upload' do let(:remote_file) { double(create: '') } @@ -23,9 +67,11 @@ it 'sends the file to the remote repository' do expect(remote_file).to receive(:create).with( - key: 'db/data/backup_file.sql', - body: 'file body', - tags: 'production-backup' + { + key: 'db/data/backup_file.sql', + body: 'file body', + tags: 'production-backup' + } ) subject.upload('/rails-app/db/backups/backup_file.sql', 'production-backup')