'Amazon S3 (v2)',
'description' => 'Amazon S3 is a well known cloud storage provider. This destination is known to be reliable and works well with BackupBuddy. Supports both bursting and chunking. Learn more here.',
);
private static $_client = '';
private static $_client_signature = '';
private static $_timeStart = 0;
private static $_chunksSentThisRound = 0;
// Default settings. Should be public static for auto-merging.
public static $default_settings = array(
'type' => 's32', // MUST MATCH your destination slug. Required destination field.
'title' => '', // Required destination field.
'accesskey' => '', // Amazon access key.
'secretkey' => '', // Amazon secret key.
'bucket' => '', // Amazon bucket to put into.
'directory' => '', // Subdirectory to put into in addition to the site url directory.
'ssl' => '1', // Whether or not to use SSL encryption for connecting.
'server_encryption' => 'AES256', // Encryption (if any) to have the destination enact. Empty string for none.
'max_time' => '', // Default max time in seconds to allow a send to run for. Set to 0 for no time limit. Aka no chunking.
'max_burst' => '10', // Max size in mb of each burst within the same page load.
'db_archive_limit' => '10', // Maximum number of db backups for this site in this directory for this account. No limit if zero 0.
'full_archive_limit' => '4', // Maximum number of full backups for this site in this directory for this account. No limit if zero 0.
'files_archive_limit' => '4', // Maximum number of files only backups for this site in this directory for this account. No limit if zero 0.
'manage_all_files' => '1', // Allow user to manage all files in S3? If enabled then user can view all files after entering their password. If disabled the link to view all is hidden.
'region' => 's3.amazonaws.com', // Endpoint. Incorrectly named region here.
'storage' => 'standard', // Whether to use standard or reduced redundancy storage. Allowed values: STANDARD, REDUCED_REDUNDANCY
'use_packaged_cert' => '0', // When 1, use the packaged cacert.pem file included with the AWS SDK.
'disable_file_management' => '0', // When 1, _manage.php will not load which renders remote file management DISABLED.
//'skip_bucket_prepare' => '0', // when 1, we will skip creating the bucket and making sure it exists before trying to place files.
'disabled' => '0', // When 1, disable this destination.
'stash_mode' => '0', // When 1, this destination is wrapped with Stash.
'max_filelist_keys' => '250', // Maximum number of files to list from server via listObjects calls.
// Do not store these for destination settings. Only used to pass to functions in this file.
'_multipart_id' => '', // Instance var. Internal use only for continuing a chunked upload.
'_multipart_partnumber' => 0, // Instance var. Part number to upload next.
'_multipart_etag_parts' => array(), // Instance var. Etags for sent parts.
'_multipart_file' => '', // Instance var. Internal use only to store the file that is currently set to be multipart chunked.
'_multipart_remotefile' => '', // Instance var. Internal use only to store the remote filepath & file.
'_multipart_counts' => array(), // Instance var. Multipart chunks to send. Generated by S3's get_multipart_counts().
'_multipart_transferspeeds' => array(), // Instance var.
'_multipart_backup_type' => '', // Instance var. Type: full, db, files
'_multipart_backup_size' => '', // Instance var. Total file size in bytes.
'_retry_stash_confirm' => false, // Whether or not we need to retry confirming the file has made it to Stash.
);
/* _init()
*
* Load SDK, create S3 client, prepare bucket, format $settings & return settings.
*
* @param
* @return array Array of formatted and sanitized settings.
*/
private static function _init( $settings ) {
pb_backupbuddy::status( 'details', 'Loading AWS SDK in _init.' );
require_once( dirname( dirname( __FILE__ ) ) . '/_s3lib2/aws-autoloader.php' );
pb_backupbuddy::status( 'details', 'SDK loaded.' );
$settings = self::_formatSettings( $settings ); // Format all settings.
/*
echo '';
print_r( $settings );
echo '
';
*/
// If not connected with these exact settings (by comparisong signatue of $settings ) then connect & prepare bucket.
//if ( ! isset( self::$_client ) ) {
$newSignature = md5( serialize( $settings ) );
if ( $newSignature != self::$_client_signature ) {
self::$_client_signature = md5( serialize( $settings ) );
// Base credentials.
$s3Config = self::getCredentials( $settings );
// SSL option.
if ( '0' == $settings['ssl'] ) {
$s3Config['scheme'] = 'http';
pb_backupbuddy::status( 'details', 'SSL disabled.' );
}
// Proxy (if applicable)
if ( defined( 'WP_PROXY_HOST' ) ) {
pb_backupbuddy::status( 'details', 'WordPress proxy setting detecred since WP_PROXY_HOST defined.' );
if ( ! is_array( $s3config['request.options'] ) ) {
$s3config['request.options'] = array();
}
$s3config['request.options']['proxy'] = WP_PROXY_HOST;
if ( defined( 'WP_PROXY_PORT' ) ) {
$s3config['request.options']['proxy'] .= ':' . WP_PROXY_PORT;
}
pb_backupbuddy::status( 'details', 'Calculated proxy URL (before user/pass added): `' . $s3config['request.options']['proxy'] . '`.' );
if ( defined( 'WP_PROXY_USERNAME' ) ) {
$s3config['request.options']['proxy'] = WP_PROXY_USERNAME . ':' . WP_PROXY_PASSWORD . '@' . $s3config['request.options']['proxy'];
}
}
$s3Config['signature'] = 'v4';
$s3Config['region'] = str_replace( array( '.amazonaws.com', 's3-' ), '', $settings['region'] );
if ( 's3' == $s3Config['region'] ) {
$s3Config['region'] = 'us-east-1';
}
// Cannot use this since we STILL need to know the correct region so that v4 signature signing can occur. Catch-22.
//$s3Config['endpoint'] = 'https://' . $settings['bucket'] . '.s3.amazonaws.com';
//pb_backupbuddy::status( 'details', 'Using specified region results in endpoint: `' . $s3Config['endpoint'] . '` to connect to.' );
/*
echo 's3config:';
print_r( $s3Config );
echo '
';
*/
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
//error_log( print_r( $s3Config, true ) );
}
self::$_client = S3Client::factory( $s3Config );
self::$_client->getConfig()->set( 'curl.options', array( 'body_as_string' => true ) ); // Work around "[curl] 65: necessary data rewind wasn't possible" issue. See https://github.com/aws/aws-sdk-php/issues/284
// Verify bucket exists; create if not. Also set region to the region bucket exists in.
/*
$prepareBucket = false;
if ( ( true === $prepareBucket ) && ( false === self::_prepareBucketAndRegion( $settings ) ) ) {
return self::_error( 'Error #983483437: Could not prepare bucket `' . $settings['bucket'] . '` in region `' . $settings['region'] . '`.' );
} else {
pb_backupbuddy::status( 'details', 'Skipping bucket prepare.' );
}
*/
}
return $settings; // Formatted & updated settings.
} // End _init().
/* _formatSettings()
*
* Called by _formatSettings().
*
*/
public static function _formatSettings( $settings ) {
// Apply defaults.
$settings = array_merge( self::$default_settings, $settings );
// Format bucket.
$settings['bucket'] = strtolower( $settings['bucket'] );
// Format directory.
$settings['directory'] = trim( $settings['directory'], '/\\' );
if ( $settings['directory'] != '' ) {
$settings['directory'] .= '/';
}
return $settings;
} // End _formatSettings().
/* send()
*
* Send one or more files.
*
* @param array $file Array of one or more files to send.
* @return boolean|array True on success, false on failure, array if a multipart chunked send so there is no status yet.
*/
public static function send( $settings = array(), $file, $send_id = '', $delete_after = false ) {
global $pb_backupbuddy_destination_errors;
if ( '1' == $settings['disabled'] ) {
$pb_backupbuddy_destination_errors[] = __( 'Error #48933: This destination is currently disabled. Enable it under this destination\'s Advanced Settings.', 'it-l10n-backupbuddy' );
return false;
}
$settings = self::_init( $settings ); // Handles formatting & sanitizing settings.
$chunkSizeBytes = $settings['max_burst'] * 1024 * 1024; // Send X mb at a time to limit memory usage.
self::$_timeStart = microtime( true );
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Settings due to log level: `' . print_r( $settings, true ) . '`.' );
}
// Initiate multipart upload.
if ( '' == $settings['_multipart_id'] ) { // New transfer. Note: All transfers are handled as presumed multiparts for ease.
// Handle chunking of file into a multipart upload (if applicable).
$file_size = filesize( $file );
pb_backupbuddy::status( 'details', 'File size of `' . pb_backupbuddy::$format->file_size( $file_size ) . '`.' );
if ( '1' != $settings['stash_mode'] ) {
// About to chunk so cleanup any previous hanging multipart transfers.
self::multipart_cleanup( $settings );
}
// Initiate multipart upload with S3.
pb_backupbuddy::status( 'details', 'Initiating multipart transfer.' );
$thisCall = array(
'Bucket' => $settings['bucket'],
'Key' => $settings['directory'] . basename( $file ),
'StorageClass' => $settings['storage'],
'ServerSideEncryption' => 'AES256',
);
if ( '1' == $settings['stash_mode'] ) {
$thisCall['Key'] = $settings['_stash_object'];
unset( $thisCall['StorageClass'] );
}
try {
$response = self::$_client->createMultipartUpload( $thisCall );
} Catch( Exception $e ) {
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Call details due to logging level: `' . print_r( $thisCall, true ) . '`.' );
}
return self::_error ( 'Error #389383: Unable to initiate multipart upload. Details: `' . $e->getMessage() . '`.' );
}
// Made it here so SUCCESS initiating multipart!
$upload_id = (string) $response['UploadId'];
pb_backupbuddy::status( 'details', 'Initiated multipart upload with ID `' . $upload_id . '`.' );
$backup_type = backupbuddy_core::getBackupTypeFromFile( $file );
// Calculate multipart settings.
$multipart_destination_settings = $settings;
$multipart_destination_settings['_multipart_id'] = $upload_id;
$multipart_destination_settings['_multipart_partnumber'] = 0;
$multipart_destination_settings['_multipart_file'] = $file;
$multipart_destination_settings['_multipart_remotefile'] = $settings['directory'] . basename( $file );
if ( '1' == $settings['stash_mode'] ) {
$multipart_destination_settings['_multipart_remotefile'] = $settings['_stash_object'];
}
$multipart_destination_settings['_multipart_counts'] = self::_get_multipart_counts( $file_size, $settings['max_burst'] * 1024 * 1024 ); // Size of chunks expected to be in bytes.
$multipart_destination_settings['_multipart_backup_type'] = $backup_type;
$multipart_destination_settings['_multipart_backup_size'] = $file_size;
$multipart_destination_settings['_multipart_etag_parts'] = array();
//pb_backupbuddy::status( 'details', 'Multipart settings to pass:' . print_r( $multipart_destination_settings, true ) );
//$multipart_destination_settings['_multipart_status'] = 'Starting send of ' . count( $multipart_destination_settings['_multipart_counts'] ) . ' parts.';
pb_backupbuddy::status( 'details', 'Multipart initiated; passing over to send first chunk this run.' );
$settings = $multipart_destination_settings; // Copy over settings.
unset( $multipart_destination_settings );
} // end initiating multipart.
// Send parts.
$backup_type = str_replace( '/', '', $settings['_multipart_backup_type'] ); // For use later by file limiting.
$backup_size = $settings['_multipart_backup_size'];
$maxTime = $settings['max_time'];
if ( ( '' == $maxTime ) || ( ! is_numeric( $maxTime ) ) ) {
pb_backupbuddy::status( 'details', 'Max time not set in settings so detecting server max PHP runtime.' );
$maxTime = backupbuddy_core::detectMaxExecutionTime();
}
pb_backupbuddy::status( 'details', 'Using max runtime: `' . $maxTime . '`.' );
// Open file for streaming.
$f = @fopen( $settings['_multipart_file'], 'r' );
if ( false === $f ) {
return self::_error( 'Error #437734. Unable to open file `' . $settings['_multipart_file'] . '` to send. Did it get deleted?' );
}
$fileDone = false;
while ( ( ! $fileDone ) && ( ! feof( $f ) ) ) {
$sendStart = microtime( true );
if ( ! isset( $settings[ '_retry_stash_confirm' ] ) || ( true !== $settings[ '_retry_stash_confirm' ] ) ) { // Skip send if only needing to confirm.
// Made it here so success sending part. Increment for next part to send.
$settings['_multipart_partnumber']++;
if ( ! isset( $settings['_multipart_counts'][ ( $settings['_multipart_partnumber'] -1 ) ]['seekTo'] ) ) {
pb_backupbuddy::status( 'error', 'Error #8239933: Missing multipart partnumber to seek to. Settings array: `' . print_r( $settings, true ) . '`.' );
}
if ( -1 == ( fseek( $f, (integer) $settings['_multipart_counts'][ ( $settings['_multipart_partnumber'] - 1 ) ]['seekTo'] ) ) ) {
return self::_error( 'Error #833838: Unable to fseek file.' );
}
pb_backupbuddy::status( 'details', 'Beginning upload of part `' . $settings['_multipart_partnumber'] . '` of `' . count( $settings['_multipart_counts'] ) . '` parts of file `' . $settings['_multipart_file'] . '` to remote location `' . $settings['_multipart_remotefile'] . '` with multipart ID `' . $settings['_multipart_id'] . '`.' );
$contentLength = (integer) $settings['_multipart_counts'][ ( $settings['_multipart_partnumber'] - 1 ) ]['length'];
$uploadArr = array(
'Bucket' => $settings['bucket'],
'Key' => $settings['_multipart_remotefile'],
'UploadId' => $settings['_multipart_id'],
'PartNumber' => $settings['_multipart_partnumber'],
'ContentLength' => $contentLength,
'Body' => fread( $f, $contentLength ), //$f,
);
//pb_backupbuddy::status( 'details', 'Send array: `' . print_r( $uploadArr, true ) . '`.' );
//error_log( print_r( $uploadArr, true ) );
try {
$response = self::$_client->uploadPart( $uploadArr );
} Catch( Exception $e ) {
@fclose( $f );
return self::_error( 'Error #3897923: Unable to upload file part for multipart upload of ID `' . $settings['_multipart_id'] . '`. Details: `' . $e->getMessage() . '`.' );
}
self::$_chunksSentThisRound++;
$settings['_multipart_etag_parts'][] = array(
'PartNumber' => $settings['_multipart_partnumber'],
'ETag' => $response['ETag'],
);
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Success sending chunk. Upload details due to log level: `' . print_r( $response, true ) . '`.' );
} else {
pb_backupbuddy::status( 'details', 'Success sending chunk. Enable full logging for upload result details.' );
}
$uploaded_size = $contentLength;
$elapseTime = ( microtime(true) - $sendStart );
if ( 0 == $elapseTime ) {
$elapseTime = 1;
}
$uploaded_speed = $uploaded_size / $elapseTime;
pb_backupbuddy::status( 'details', 'Uploaded size this burst: `' . pb_backupbuddy::$format->file_size( $uploaded_size ) . '`, Start time: `' . $sendStart . '`. Finish time: `' . microtime(true) . '`. Elapsed: `' . (microtime(true) - $sendStart ) . '`. Speed: `' . pb_backupbuddy::$format->file_size( $uploaded_speed ) . '`/sec.' );
}
// Load fileoptions to the send.
if ( isset( $fileoptions_obj ) ) {
pb_backupbuddy::status( 'details', 'fileoptions already loaded from prior pass.' );
} else { // load fileoptions
pb_backupbuddy::status( 'details', 'About to load fileoptions data.' );
require_once( pb_backupbuddy::plugin_path() . '/classes/fileoptions.php' );
pb_backupbuddy::status( 'details', 'Fileoptions instance #10.' );
$fileoptions_obj = new pb_backupbuddy_fileoptions( backupbuddy_core::getLogDirectory() . 'fileoptions/send-' . $send_id . '.txt', $read_only = false, $ignore_lock = false, $create_file = false );
if ( true !== ( $result = $fileoptions_obj->is_ok() ) ) {
return self::_error( __('Fatal Error #9034.23788723. Unable to access fileoptions data.', 'it-l10n-backupbuddy' ) . ' Error: ' . $result );
}
pb_backupbuddy::status( 'details', 'Fileoptions data loaded.' );
$fileoptions = &$fileoptions_obj->options;
}
//$update_status = 'Sent part ' . $settings['_multipart_partnumber'] . ' of ' . count( $settings['_multipart_counts'] ) . '.';
if ( ! isset( $settings['_multipart_counts'][ $settings['_multipart_partnumber'] ] ) ) { // No more parts exist for this file. Tell S3 the multipart upload is complete and move on.
if ( isset( $settings[ '_retry_stash_confirm' ] ) && ( true === $settings[ '_retry_stash_confirm' ] ) ) { // Need to retry checking that the file confirm was a success.
// Grab array of files from customer's stash directory
$files = pb_backupbuddy_destination_stash2::listFiles( $settings, $settings['_multipart_file'] );
if ( count( $files ) > 0 ) {
pb_backupbuddy::status( 'details', 'Stash confirmed upload completition was successful.' );
} else {
pb_backupbuddy::status( 'error', 'Error #23972793: Error notifying Stash of upload success even after wait. Details: `' . print_r( $response, true ) . '`.' );
return false;
}
} else { // Normal Stash part send.
$update_status = 'Sent part ' . $settings['_multipart_partnumber'] . ' of ' . count( $settings['_multipart_counts'] ) . ' parts.';
pb_backupbuddy::status( 'details', 'Getting etags and notifying of multipart upload completion.' );
try {
$response = self::$_client->completeMultipartUpload( array(
'Bucket' => $settings['bucket'],
'UploadId' => $settings['_multipart_id'],
'Key' => $settings['_multipart_remotefile'],
'Parts' => $settings['_multipart_etag_parts']
) );
} Catch( Exception $e ) {
return self::_error( 'Unable to notify server of completion of all parts for multipart upload `' . $settings['_multipart_id'] . '`. Details: `' . $e->getMessage() . '`.' );
}
pb_backupbuddy::status( 'details', 'Server notified of multipart completion.' );
if ( '1' == $settings['stash_mode'] ) { // Stash send confirm.
pb_backupbuddy::status( 'details', 'Notifying Stash of upload completion.' );
$additionalParams =array(
'upload_id' => $settings['_stash_upload_id'],
);
$response = pb_backupbuddy_destination_stash2::stashAPI( $settings, 'upload-complete', $additionalParams );
if ( ( ! is_array( $response ) ) || ( ! isset( $response['success'] ) ) ) { // If not array OR success key missing. May be a timeout or waiting on AWS system to combine multipart still. Check for file later.
$settings[ '_retry_stash_confirm' ] = true;
$settings['_multipart_counts'] = array(); // No more parts remain.
$cronTime = time() + self::STASH_CONFIRM_RETRY_DELAY;
$cronArgs = array( $settings, $file, $send_id, $delete_after );
$cronHashID = md5( $cronTime . serialize( $cronArgs ) );
$cronArgs[] = $cronHashID;
$schedule_result = backupbuddy_core::schedule_single_event( $cronTime, 'destination_send', $cronArgs );
if ( true === $schedule_result ) {
pb_backupbuddy::status( 'details', 'Scheduled retry attempt to confirm send in `' . self::STASH_CONFIRM_RETRY_DELAY . '` seconds.' );
} else {
pb_backupbuddy::status( 'error', 'Scheduled retry attempt FAILED to be scheduled.' );
}
/*
* TODO: Once PING API is available, request a ping in the future so we make sure this actually runs reasonably soon.
* Because we need a delay we are not firing off the cron here immediately so there will be no chaining of PHP
* which may result in large delays before the next process if there's little site traffic.
*/
return array( $settings['_multipart_id'], 'Pending multipart send confirmation.' );
} else { // Array.
if ( ( isset( $response['success'] ) && ( true !== $response['success'] ) ) ) { // Success key set AND not true.
pb_backupbuddy::status( 'error', 'Error #83298932: Error notifying Stash of upload success. Details: `' . print_r( $response, true ) . '`.' );
return false;
} else { // Success.
pb_backupbuddy::status( 'details', 'Stash notified of upload completition.' );
}
}
}
} // end not a Stash confirm retry.
pb_backupbuddy::status( 'details', 'No more parts left for this multipart upload. Clearing multipart instance variables.' );
$settings['_multipart_partnumber'] = 0;
$settings['_multipart_id'] = '';
$settings['_multipart_file'] = '';
$settings['_multipart_remotefile'] = ''; // Multipart completed so safe to prevent housekeeping of incomplete multipart uploads.
$settings['_multipart_transferspeeds'][] = $uploaded_speed;
// Overall upload speed average.
$uploaded_speed = array_sum( $settings['_multipart_transferspeeds'] ) / count( $settings['_multipart_counts'] );
pb_backupbuddy::status( 'details', 'Upload speed average of all chunks: `' . pb_backupbuddy::$format->file_size( $uploaded_speed ) . '`.' );
$settings['_multipart_counts'] = array();
// Update stats.
$fileoptions['_multipart_status'] = $update_status;
$fileoptions['finish_time'] = microtime( true );
$fileoptions['status'] = 'success';
if ( isset( $uploaded_speed ) ) {
$fileoptions['write_speed'] = $uploaded_speed;
}
$fileoptions_obj->save();
unset( $fileoptions );
$fileDone = true;
@fclose( $f );
} else { // Parts remain. Schedule to continue if anything is left to upload for this multipart of any individual files.
pb_backupbuddy::status( 'details', 'S3 multipart upload has more parts left.' );
$update_status = '
';
$totalSent = 0;
for( $i = 0; $i < $settings['_multipart_partnumber']; $i++ ) {
$totalSent += $settings['_multipart_counts'][ $i ]['length'];
}
$percentSent = ceil( ( $totalSent / $settings['_multipart_backup_size'] ) * 100 );
$update_status .= '';
if ( '0' != $maxTime ) { // Not unlimited time so see if we can send more bursts this time or if we need to chunk.
// If we are within X second of reaching maximum PHP runtime then stop here so that it can be picked up in another PHP process...
$totalSizeSent = self::$_chunksSentThisRound * $chunkSizeBytes; // Total bytes sent this PHP load.
$bytesPerSec = $totalSizeSent / ( microtime( true ) - $sendStart );
$timeRemaining = $maxTime - ( ( microtime( true ) - self::$_timeStart ) + self::TIME_WIGGLE_ROOM );
if ( $timeRemaining < 0 ) {
$timeRemaining = 0;
}
$bytesWeCouldSendWithTimeLeft = $bytesPerSec * $timeRemaining;
//pb_backupbuddy::status( 'details', 'Sent this burst: `' . pb_backupbuddy::$format->file_size( $totalSizeSent ) .'` in `' . (microtime(true) - $sendStart ) . '` secs. Speed: `' . pb_backupbuddy::$format->file_size( $bytesPerSec ) . '`/sec. Time Remaining (w/ wiggle): `' . $timeRemaining . '`. Size that could potentially be sent with remaining time: `' . pb_backupbuddy::$format->file_size( $bytesWeCouldSendWithTimeLeft ) . '` with chunk size of `' . pb_backupbuddy::$format->file_size( $chunkSizeBytes ) . '`.' );
if ( $bytesWeCouldSendWithTimeLeft < $chunkSizeBytes ) { // We can send more than a whole chunk (including wiggle room) so send another bit.
pb_backupbuddy::status( 'message', 'Not enough time left (~`' . $timeRemaining . '`) with max time of `' . $maxTime . '` sec to send another chunk at `' . pb_backupbuddy::$format->file_size( $bytesPerSec ) . '` / sec. Ran for ' . round( microtime( true ) - self::$_timeStart, 3 ) . ' sec. Proceeding to use chunking.' );
@fclose( $fs );
$cronTime = time();
$cronArgs = array( $settings, $file, $send_id, $delete_after );
$cronHashID = md5( $cronTime . serialize( $cronArgs ) );
$cronArgs[] = $cronHashID;
$schedule_result = backupbuddy_core::schedule_single_event( $cronTime, 'destination_send', $cronArgs );
if ( true === $schedule_result ) {
pb_backupbuddy::status( 'details', 'Next S3 chunk step cron event scheduled.' );
} else {
pb_backupbuddy::status( 'error', 'Next S3 chunk step cron even FAILED to be scheduled.' );
}
spawn_cron( time() + 150 ); // Adds > 60 seconds to get around once per minute cron running limit.
update_option( '_transient_doing_cron', 0 ); // Prevent cron-blocking for next item.
@fclose( $f );
unset( $fileoptions );
return array( $settings['_multipart_id'], 'Sent part ' . $settings['_multipart_partnumber'] . ' of ' . count( $settings['_multipart_counts'] ) . ' parts.' . $update_status );
} else { // End if.
pb_backupbuddy::status( 'details', 'Not approaching limits. Proceeding to next burst this run.' );
}
} else {
pb_backupbuddy::status( 'details', 'Max time of zero (0) so assuming unlimited time.' );
}
$fileoptions['_multipart_status'] = 'Sent part ' . $settings['_multipart_partnumber'] . ' of ' . count( $settings['_multipart_counts'] ) . ' parts.' . $update_status;
$fileoptions_obj->save();
//unset( $fileoptions );
} // end no more parts remain.
} // End while not feof.
/***** BEGIN FILE ARCHIVE LIMITS *****/
if ( '1' == $settings['stash_mode'] ) { // This is being wrapped by the Stash destination. Stash uses a different method of handling archive limiting due to using Stash API.
pb_backupbuddy_destination_stash2::archiveLimit( $settings, $backup_type );
} else { // Normal. This is just a s32 destination.
self::archiveLimit( $settings, $backup_type );
}
/***** END FILE ARCHIVE LIMITS *****/
if ( isset( $fileoptions_obj ) ) {
unset( $fileoptions_obj );
}
// Success if we made it this far.
return true;
} // End send().
/* archiveLimit()
*
* Return null.
*
*/
public static function archiveLimit( $settings, $backup_type ) {
if ( $backup_type == 'full' ) {
$limit = $settings['full_archive_limit'];
pb_backupbuddy::status( 'details', 'Full backup archive limit of `' . $limit . '` of type `full` based on destination settings.' );
} elseif ( $backup_type == 'db' ) {
$limit = $settings['db_archive_limit'];
pb_backupbuddy::status( 'details', 'Database backup archive limit of `' . $limit . '` of type `db` based on destination settings.' );
} elseif ( $backup_type == 'files' ) {
$limit = $settings['files_archive_limit'];
pb_backupbuddy::status( 'details', 'Database backup archive limit of `' . $limit . '` of type `files` based on destination settings.' );
} else {
$limit = 0;
pb_backupbuddy::status( 'warning', 'Warning #237332. Unable to determine backup type (reported: `' . $backup_type . '`) so archive limits NOT enforced for this backup.' );
}
if ( $limit > 0 ) {
pb_backupbuddy::status( 'details', 'Archive limit enforcement beginning.' );
// Get file listing.
try {
$response_manage = self::$_client->listObjects( array(
'Bucket' => $settings['bucket'],
'Prefix' => $settings['directory'] . 'backup-' . backupbuddy_core::backup_prefix()
) ); // List all users files in this directory that are a backup for this site (limited by prefix).
} Catch( Exception $e ) {
return self::_error( 'Error #9338292: Unable to list files for archive limiting. Details: `' . $e->getMessage() . '`.' );
}
if ( ! is_array( $response_manage['Contents'] ) ) {
$response_manage['Contents'] = array();
}
// List backups associated with this site by date.
$backups = array();
foreach( $response_manage['Contents'] as $object ) {
$file = str_replace( $settings['directory'], '', $object['Key'] );
if ( $backup_type != backupbuddy_core::getBackupTypeFromFile( $file, $quiet = true ) ) {
continue; // Not of the same backup type.
}
$backups[$file] = strtotime( $object['LastModified'] );
}
arsort( $backups );
pb_backupbuddy::status( 'details', 'Found `' . count( $backups ) . '` backups of this type when checking archive limits out of `' . count( $response_manage['Contents'] ) . '` total files in this location.' );
if ( ( count( $backups ) ) > $limit ) {
pb_backupbuddy::status( 'details', 'More archives (' . count( $backups ) . ') than limit (' . $limit . ') allows. Trimming...' );
$i = 0;
$delete_fail_count = 0;
foreach( $backups as $buname => $butime ) {
$i++;
if ( $i > $limit ) {
pb_backupbuddy::status( 'details', 'Trimming excess file `' . $buname . '`...' );
try {
$response = self::$_client->deleteObject( array(
'Bucket' => $settings['bucket'],
'Key' => $settings['directory'] . $buname,
) );
} Catch( Exception $e ) {
self::_error( 'Unable to delete excess Stash file `' . $buname . '`. Details: `' . $e->getMessage() . '`.' );
$delete_fail_count++;
}
}
} // end foreach.
pb_backupbuddy::status( 'details', 'Finished trimming excess backups.' );
if ( $delete_fail_count !== 0 ) {
$error_message = 'Stash remote limit could not delete ' . $delete_fail_count . ' backups.';
pb_backupbuddy::status( 'error', $error_message );
backupbuddy_core::mail_error( $error_message );
}
}
pb_backupbuddy::status( 'details', 'Stash completed archive limiting.' );
} else {
pb_backupbuddy::status( 'details', 'No Stash archive file limit to enforce.' );
} // End remote backup limit
return true;
} // End archiveLimit().
/* listFiles()
*
* Returns array of files or string error message.
*
*/
public static function listFiles( $settings, $prefix = '', $marker = '' ) {
$settings = self::_init( $settings );
/*
echo '
Prefix: ' . $prefix . '
';
echo 'listfilessettings:';
print_r( $settings );
echo '
';
*/
$options = array(
'Bucket' => $settings['bucket'],
'Prefix' => $prefix
);
if ( isset( $settings['max_filelist_keys'] ) ) {
$options['MaxKeys'] = $settings['max_filelist_keys'];
}
if ( '' !== $marker ) {
$options['Marker'] = $marker;
}
try {
$response = self::$_client->listObjects( $options ); // list all the files in the subscriber account.
} Catch( Exception $e ) {
$error = 'Error #838393: Unable to list files. Details: `' . $e->getMessage() . '`. Bucket: `' . $settings['bucket'] . '`. Prefix: `' . $prefix . '`.';
if ( stristr( $error, 'Please send all future requests to this endpoint' ) ) {
$error .= ' TO FIX THIS: Navigate to the settings for this destination and change the `Bucket region` to the correct location.';
}
self::_error( $error );
return $error;
}
/*
echo ' listfilesResponse:';
print_r( $response );
echo '
';
*/
return (array)$response['Contents'];
} // End listFiles().
/* deleteFile()
*
* Alias to deleteFiles().
*
*/
public static function deleteFile( $settings, $file ) {
if ( ! is_array( $file ) ) {
$file = array( $file );
}
return self::deleteFiles( $settings, $file );
} // End deleteFile().
/* deleteFiles()
*
* true success deleting all files else string error message.
*
*/
public static function deleteFiles( $settings, $files = array() ) {
$settings = self::_init( $settings );
if ( ! is_array( $files ) ) {
$file = array( $files );
}
$fileKeys = array();
foreach( (array)$files as $file ) {
$fileKeys[] = array(
'Key' => $settings['directory'] . $file
);
}
try {
$response = self::$_client->deleteObjects( array(
'Bucket' => $settings['bucket'],
'Objects' => $fileKeys
) ); // list all the files in the subscriber account
} Catch( Exception $e ) {
$error = 'Error #83823233393: Unable to delete one or more files. Details: `' . $e->getMessage() . '`.';
self::_error( $error );
return $error;
}
$error = '';
if ( isset( $response['Errors'] ) ) {
foreach( $response['Errors'] as $responseError ) {
$error .= $responseError['Message'];
}
pb_backupbuddy::status( 'error', $error );
return $error;
}
return true;
} // End deleteFiles().
/* test()
*
* Tests ability to write to this remote destination.
*
* @param array $settings Destination settings.
* @return bool|string True on success, string error message on failure.
*/
public static function test( $settings ) {
$settings = self::_init( $settings );
$sendOK = false;
$deleteOK = false;
$send_id = 'TEST-' . pb_backupbuddy::random_string( 12 );
// Try sending a file.
if ( '1' == $settings['stash_mode'] ) { // Stash mode.
$settings['type'] = 'stash2';
}
$send_response = pb_backupbuddy_destinations::send( $settings, dirname( dirname( __FILE__ ) ) . '/remote-send-test.php', $send_id ); // 3rd param true forces clearing of any current uploads.
if ( true === $send_response ) {
$send_response = __( 'Success.', 'it-l10n-backupbuddy' );
$sendOK = true;
} else {
global $pb_backupbuddy_destination_errors;
$send_response = 'Error sending test file to S3 (v2). Details: `' . implode( ', ', $pb_backupbuddy_destination_errors ) . '`.';
}
pb_backupbuddy::add_status_serial( 'remote_send-' . $send_id );
// Delete sent file if it was sent.
$delete_response = 'n/a';
if ( true === $sendOK ) {
pb_backupbuddy::status( 'details', 'Preparing to delete sent test file.' );
if ( '1' == $settings['stash_mode'] ) { // Stash mode.
if ( true === ( $delete_response = pb_backupbuddy_destination_stash2::deleteFile( $settings, 'remote-send-test.php' ) ) ) { // success
$delete_response = __( 'Success.', 'it-l10n-backupbuddy' );
$deleteOK = true;
} else { // error
$error = 'Unable to delete Stash test file `remote-send-test.php`. Details: `' . $delete_response . '`.';
$delete_response = $error;
$deleteOK = false;
}
} else { // S3 mode.
if ( true === ( $delete_response = self::deleteFile( $settings, 'remote-send-test.php' ) ) ) {
$delete_response = __( 'Success.', 'it-l10n-backupbuddy' );
$deleteOK = true;
} else {
$error = 'Unable to delete test file `remote-send-test.php`. Details: `' . $delete_response . '`.';
pb_backupbuddy::status( 'details', $error );
$delete_response = $error;
$deleteOK = false;
}
}
} else { // end if $sendOK.
pb_backupbuddy::status( 'details', 'Skipping test delete due to failed send.' );
}
// Load destination fileoptions.
pb_backupbuddy::status( 'details', 'About to load fileoptions data.' );
require_once( pb_backupbuddy::plugin_path() . '/classes/fileoptions.php' );
pb_backupbuddy::status( 'details', 'Fileoptions instance #7.' );
$fileoptions_obj = new pb_backupbuddy_fileoptions( backupbuddy_core::getLogDirectory() . 'fileoptions/send-' . $send_id . '.txt', $read_only = false, $ignore_lock = false, $create_file = false );
if ( true !== ( $result = $fileoptions_obj->is_ok() ) ) {
return self::_error( __('Fatal Error #9034.84838. Unable to access fileoptions data.', 'it-l10n-backupbuddy' ) . ' Error: ' . $result );
}
pb_backupbuddy::status( 'details', 'Fileoptions data loaded.' );
$fileoptions = &$fileoptions_obj->options;
if ( ( true !== $sendOK ) || ( true !== $deleteOK ) ) {
$fileoptions['status'] = 'failure';
$fileoptions_obj->save();
unset( $fileoptions_obj );
return 'Send details: `' . $send_response . '`. Delete details: `' . $delete_response . '`.';
} else {
$fileoptions['status'] = 'success';
$fileoptions['finish_time'] = time();
}
$fileoptions_obj->save();
unset( $fileoptions_obj );
pb_backupbuddy::status( 'details', 'Finished test function.' );
return true;
} // End test().
/* download_file()
*
* Download remote file to local system.
*
* @param array $settings Destination settings.
* @param string $remoteFile Remote filename.
* @param string $localDestinationFile Full path & filename of destination file.
*
*/
public static function download_file( $settings, $remoteFile, $localDestinationFile ) {
$settings = self::_init( $settings );
pb_backupbuddy::status( 'details', 'Downloading remote file `' . $remoteFile . '` from S3 to local file `' . $localDestinationFile . '`.' );
try {
$response = self::$_client->get_object( array(
'Bucket' => $settings['bucket'],
'Key' => $remotePath . $remoteFile,
'SaveAs' => $localDestinationFile
) );
} Catch( Exception $e ) {
return self::_error( 'Error #382938: Unable to retrieve file. Details: `' . $e->getMessage() . '`.' );
}
return true;
} // end download_file().
/* getFileURL()
*
* Get download URL for a file.
*
* @param array $settings Destination settings.
* @param string $remoteFile Filename of remote file. Does NOT contain path as it is calculated from $settings.
* @param int $expires Timestamp to expire. Defaults to 1 hour.
* @return
*
*/
public static function getFileURL( $settings, $remoteFile, $expires = 0 ) {
$settings = self::_init( $settings );
pb_backupbuddy::status( 'details', 'Getting download URL.' );
if ( ( 0 == $expires ) || ( ! is_numeric( $expires ) ) ) {
$expires = time() + 3600; // 1 hour default.
}
try {
$response = self::$_client->getObjectUrl( $settings['bucket'], $settings['directory'] . $remoteFile, $expires );
} Catch( Exception $e ) {
return self::_error( 'Error #349734634: Unable to retrieve file URL. Details: `' . $e->getMessage() . '`.' );
}
return $response;
} // End getFileURL().
/* getCredentials()
*
* Get the required credentials and management data for managing user files.
*
* @return false|array Boolean false on failure. Array of data on success.
*/
public static function getCredentials( $settings ) {
$settings['bucket'] = strtolower( $settings['bucket'] ); // Buckets must be lowercase.
if ( '1' == $settings['use_packaged_cert'] ) {
pb_backupbuddy::status( 'details', 'Using packaged cacert.pem file based on destination settings.' );
$credentials['ssl.certificate_authority'] = pb_backupbuddy::plugin_path() . '/destinations/_s3lib2/Guzzle/Http/Resources/cacert.pem';
}
if ( isset( $settings['credentials'] ) ) {
$credentials['credentials'] = $settings['credentials'];
return $credentials;
} else {
$credentials['key'] = $settings['accesskey'];
$credentials['secret'] =$settings['secretkey'];
return $credentials;
}
} // End getCredentials().
public static function get_bucket_region( $bucket ) {
try {
$result = self::$_client->getBucketLocation(array(
'Bucket' => $bucket,
));
} catch (Exception $e) {
throw new Exception( $e->getMessage() );
}
if ( ! isset( $result->Location ) ) { // Blank means default standard location.
return 'us-east-1';
}
$location = $result->Location;
if ( 'EU' == $location ) {
$location = 'eu-west-1';
}
return $location;
}
/* multipart_cleanup()
*
* S3 does NOT automatically clean up failed or expired multipart chunk files so clean up for them.
*
*/
public static function multipart_cleanup( $settings ) {
$settings = self::_init( $settings ); // Handles formatting & sanitizing settings.
pb_backupbuddy::status( 'details', 'AWS Multipart cleanup beginning.' );
try {
$response = self::$_client->listMultipartUploads( array(
'Bucket' => $settings['bucket'],
'prefix' => 'backup',
) );
} Catch( Exception $e ) {
return self::_error( 'Error #84397849347: Unable to list existing multipart uploads. Details: `' . $e->getMessage() . '`' );
}
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Multipart upload check retrieved. Found `' . count( $response['Uploads'] ) . '` multipart uploads in progress / stalled. Full logging mode details: `' . print_r( $response, true ) . '`' );
} else {
pb_backupbuddy::status( 'details', 'Multipart upload check retrieved. Found `' . count( $response['Uploads'] ) . '` multipart uploads in progress / stalled. Old BackupBuddy parts will be cleaned up (if any found) ...' );
}
// Loop through each incomplete multipart upload.
foreach( (array)$response['Uploads'] as $upload ) {
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Checking upload (full logging mode): ' . print_r( $upload, true ) );
}
if ( FALSE !== stristr( $upload['Key'], 'backup-' ) ) { // BackupBuddy backup file.
$initiated = strtotime( $upload['Initiated'] );
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Multipart Chunked Upload(s) detected in progress. Full logging age: `' . pb_backupbuddy::$format->time_ago( $initiated ) . '`.' );
}
// If too old then cancel it.
if ( ( $initiated + self::MAX_AGE_MULTIPART_UPLOADS ) < time() ) {
pb_backupbuddy::status( 'details', 'Aborting stalled Multipart Chunked Upload with ID `' . $upload->UploadId . '` of age `' . pb_backupbuddy::$format->time_ago( $initiated ) . '`.' );
try {
self::$_client->abortMultipartUpload( array(
'Bucket' => $settings['bucket'],
'Key' => $upload['Key'],
'UploadId' => $upload['UploadId']
) );
pb_backupbuddy::status( 'details', 'Abort success.' );
} Catch( Exception $e ) {
pb_backupbuddy::status( 'error', 'Stalled Multipart Chunked abort of file `' . $upload['Key'] . '` with ID `' . $upload['UploadId'] . '` FAILED. Manually abort it. Details: `' . $e->getMessage() . '`.' );
}
} else {
if ( pb_backupbuddy::$options['log_level'] != '3' ) { // Full logging NOT enabled.
pb_backupbuddy::status( 'details', 'Multipart Chunked Uploads not aborted as not too old.' );
}
}
}
}
pb_backupbuddy::status( 'details', 'AWS Multipart cleanup finished.' );
return true;
} // end multipart_cleanup().
/* _prepareBucketAndRegion()
*
* Validates bucket existance, creating if needed. Sets region for non-US usage.
*
* @param array $settings Destination settings array.
* @param bool $createBucket Whether or not to create bucket if it does not currently exist.
* @return bool true on all okay, false otherwise.
*
*/
private static function _prepareBucketAndRegion( $settings, $createBucket = true ) {
$error = 'BackupBuddy Error #32823893: Obsolete function call.';
error_log( $error );
echo $error;
return false;
$settings = self::_formatSettings( $settings ); // Format all settings.
if ( '1' == $settings['skip_bucket_prepare'] ) {
pb_backupbuddy::status( 'details', 'Skipping bucket preparation based on destination settings.' );
return true;
}
// Get bucket region to determine if a bucket already exists.
// Assume we will not have to try and create a bucket
$maybe_create_bucket = false;
pb_backupbuddy::status( 'details', 'Getting region for bucket: `' . $settings['bucket'] . "`." );
/*
echo 'Settings:';
print_r( $settings );
echo '
';
*/
try {
$detectedRegion = self::get_bucket_region( $settings['bucket'] );
$result = self::$_client->getBucketLocation(array(
'Bucket' => $settings['bucket'],
));
pb_backupbuddy::status( 'details', 'Server indicates region: ' . $detectedRegion );
$settings['region'] = $detectedRegion; // Override passed region.
} catch( Exception $e ) {
$detectedRegion = '';
$maybe_create_bucket = true;
$message = 'Exception retrieving information for bucket `' . $settings['bucket'] . '`. Assuming region in $settings correct. If using IAM security, verify this resource ALLOWs the action "s3:GetBucketLocation". Details: `' . $e->getMessage() . '`. Full result: `' . print_r( $result, true ) . '`.';
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Settings used due to log level: `' . print_r( $settings, true ) . '`.' );
}
echo $message;
pb_backupbuddy::status( 'warning', $message );
//self::_error ( $message );
}
try {
self::$_client->setRegion( $settings['region'] );
} catch( Exception $e ) {
throw new Exception( 'Unable to set region. Details: `' . $e->getMessage() . '`.' );
}
// In bucket creation mode AND bucket did not already exist.
if ( ( true === $createBucket ) && ( true === $maybe_create_bucket ) ) {
pb_backupbuddy::status( 'details', 'Attempting to create bucket `' . $settings['bucket'] . '` at region endpoint `' . $settings['region'] . '` (detected region: `' . $detectedRegion . '`).' );
try {
$response = self::$_client->createBucket( array(
'ACL' => 'private',
'Bucket' => $settings['bucket'],
'LocationConstraint' => $settings['region']
) );
} catch( Exception $e ) {
return self::_error( 'Error #3892833: Unable to create bucket. Details: `' . $e->getMessage() . '`.' );
}
} // end if create bucket.
return true;
} // end _prepareBucketAndRegion().
// Taken from v1 s3 SDK.
public static function _get_multipart_counts($filesize, $part_size)
{
$i = 0;
$sizecount = $filesize;
$values = array();
while ($sizecount > 0)
{
$sizecount -= $part_size;
$values[] = array(
'seekTo' => ($part_size * $i),
'length' => (($sizecount > 0) ? $part_size : ($sizecount + $part_size)),
);
$i++;
}
if ( pb_backupbuddy::$options['log_level'] == '3' ) { // Full logging enabled.
pb_backupbuddy::status( 'details', 'Multipart counts due to log level: `' . print_r( $values, true ) . '`.' );
}
return $values;
}
private static function _error( $message ) {
global $pb_backupbuddy_destination_errors;
$pb_backupbuddy_destination_errors[] = $message;
pb_backupbuddy::status( 'error', $message );
return false;
}
} // End class.