Merge pull request #293 from humanmade/fix-upload-directory (09b2062e) · Commits · singing-chimes.co.uk / S3-Uploads

README.md

+13 −18

Original line number	Diff line number	Diff line
		@@ -75,18 +75,7 @@ wp s3-uploads create-iam-user --admin-key=<key> --admin-secret=<secret>

		This will provide you with a new Access Key and Secret Key which you can configure S3-Uploads with. Paste the values in the `wp-config.php`. Once you have migrated your media to S3 with any of the below methods, you'll want to enable S3 Uploads: `wp s3-uploads enable`.

		If you want to create your IAM user yourself, or attach the neccessary permissions to an existing user, you can output the policy via `wp s3-uploads generate-iam-policy`

		Migrating your Media to S3
		==========

		S3-Uploads can migrate your existing media library to S3. Once you have S3-Uploads up and running, use the following WP-CLI command:

		```
		wp s3-uploads migrate-attachments [--delete-local]
		```

		By default, S3-Uploads will keep your files locally just incase something goes wrong, but you can delete with the `--delete-local` flag.
		If you want to create your IAM user yourself, or attach the necessary permissions to an existing user, you can output the policy via `wp s3-uploads generate-iam-policy`


		Listing files on S3
		@@ -101,14 +90,20 @@ wp s3-uploads ls [<path>]
		Uploading files to S3
		==========

		Sometimes the `wp s3-uploads migrate-attachments` command may not be enough to migrate your uploads to S3, as that will only move attachment files to S3. If you are using any plugins that store data in uploads, you'll want to upload the whole `uploads` directory.
		If you have an existing media library with attachment files, use the below command to copy them all to S3 from local disk.

		```
		wp s3-uploads upload-directory <from> <to> [--sync] [--dry-run]
		wp s3-uploads upload-directory <from> <to> [--verbose]
		```

		Passing `--sync` will only upload files that are newer in `<from>` or that don't exist on S3 already. Use `--dry-run` to test.

		For example, to migrate your whole uploads directory to S3, you'd run:

		```
		wp s3-uploads upload-directory /path/to/uploads/ uploads
		```

		There is also an all purpose `cp` command for arbitrary copying to and from S3.

		```

inc/class-s3-uploads-changed-files-iterator.php

deleted100644 → 0

+0 −25

Original line number	Diff line number	Diff line
		<?php

		class S3_Uploads_ChangedFilesIterator extends Aws\S3\Sync\ChangedFilesIterator {

		public $dry_run = false;
		public function accept() {

		$current = $this->current();

		if ( ! empty( $this->dry_run ) && defined( 'WP_CLI' ) ) {
		if ( $current->getMTime() > $data[1] ) {
		WP_CLI::line( "(dry-run) Uploading {$current->getPathname()}" );
		}
		return false;
		}

		$key = $this->sourceConverter->convert( (string) $current );
		if ( ! ( $data = $this->getTargetData( $key ) ) ) {
		return true;
		}

		// Ensure it hasn't been modified since the mtime
		return $current->getMTime() > $data[1];
		}
		}

inc/class-s3-uploads-uploadsyncbuilder.php

deleted100644 → 0

+0 −44

Original line number	Diff line number	Diff line
		<?php

		class S3_Uploads_UploadSyncBuilder extends Aws\S3\Sync\UploadSyncBuilder {

		public function __construct( $is_dry_run = false ) {
		$this->dry_run = $is_dry_run;
		}

		/**
		* Builds a UploadSync or DownloadSync object
		*
		* @return AbstractSync
		*/
		public function build() {
		$this->validateRequirements();
		$this->sourceConverter = $this->sourceConverter ?: $this->getDefaultSourceConverter();
		$this->targetConverter = $this->targetConverter ?: $this->getDefaultTargetConverter();

		// Only wrap the source iterator in a changed files iterator if we are not forcing the transfers
		if ( ! $this->forcing ) {
		$this->sourceIterator->rewind();
		$this->sourceIterator = new S3_Uploads_ChangedFilesIterator(
		new \NoRewindIterator( $this->sourceIterator ),
		$this->getTargetIterator(),
		$this->sourceConverter,
		$this->targetConverter
		);
		$this->sourceIterator->dry_run = $this->dry_run;
		$this->sourceIterator->rewind();
		}

		$sync = $this->specificBuild();

		if ( $this->params ) {
		$this->addCustomParamListener( $sync );
		}

		if ( $this->debug ) {
		$this->addDebugListener( $sync, is_bool( $this->debug ) ? STDOUT : $this->debug );
		}

		return $sync;
		}
		}

inc/class-s3-uploads-wp-cli-command.php

+17 −106

Original line number	Diff line number	Diff line
		<?php

		use Aws\S3\Transfer;

		class S3_Uploads_WP_CLI_Command extends WP_CLI_Command {

		/**
		@@ -53,92 +55,6 @@ class S3_Uploads_WP_CLI_Command extends WP_CLI_Command {
		WP_CLI::success( 'Looks like your configuration is correct.' );
		}

		/**
		* @subcommand migrate-attachments
		* @synopsis [--delete-local]
		*/
		public function migrate_attachments_to_s3( $args, $args_assoc ) {

		$attachments = new WP_Query( array(
		'post_type' => 'attachment',
		'posts_per_page' => -1,
		'post_status' => 'all',
		));

		WP_CLI::line( sprintf( 'Attempting to move %d attachments to S3', $attachments->found_posts ) );

		foreach ( $attachments->posts as $attachment ) {

		$this->migrate_attachment_to_s3( array( $attachment->ID ), $args_assoc );
		}

		WP_CLI::success( 'Moved all attachments to S3. If you wish to update references in your database run: ' );
		WP_CLI::line( '' );

		// Ensure things are active
		$instance = S3_Uploads::get_instance();

		if ( ! s3_uploads_enabled() ) {
		$instance->setup();
		}

		$old_upload_dir = $instance->get_original_upload_dir();
		$upload_dir = wp_upload_dir();

		WP_CLI::Line( sprintf( 'wp search-replace "%s" "%s"', $old_upload_dir['baseurl'], $upload_dir['baseurl'] ) );
		}

		/**
		* Migrate a single attachment's files to S3
		*
		* @subcommand migrate-attachment
		* @synopsis <attachment-id> [--delete-local]
		*/
		public function migrate_attachment_to_s3( $args, $args_assoc ) {

		// Ensure things are active
		$instance = S3_Uploads::get_instance();
		if ( ! s3_uploads_enabled() ) {
		$instance->setup();
		}

		$old_upload_dir = $instance->get_original_upload_dir();
		$upload_dir = wp_upload_dir();

		$files = array();
		$attached_file = get_post_meta( $args[0], '_wp_attached_file', true );

		if ( ! empty( $attached_file ) ) {
		$files[] = $attached_file;
		}

		$meta_data = wp_get_attachment_metadata( $args[0] );

		if ( ! empty( $meta_data['sizes'] ) ) {
		foreach ( $meta_data['sizes'] as $file ) {
		$files[] = path_join( dirname( $meta_data['file'] ), $file['file'] );
		}
		}

		foreach ( $files as $file ) {
		if ( file_exists( $path = $old_upload_dir['basedir'] . '/' . $file ) ) {

		if ( ! copy( $path, $upload_dir['basedir'] . '/' . $file ) ) {
		WP_CLI::line( sprintf( 'Failed to moved %s to S3', $file ) );
		} else {
		if ( ! empty( $args_assoc['delete-local'] ) ) {
		unlink( $path );
		}
		WP_CLI::success( sprintf( 'Moved file %s to S3', $file ) );

		}
		} else {
		WP_CLI::line( sprintf( 'Already moved to %s S3', $file ) );
		}
		}

		}

		/**
		* Create an AWS IAM user for S3 Uploads to user
		*
		@@ -299,7 +215,7 @@ class S3_Uploads_WP_CLI_Command extends WP_CLI_Command {
		* Upload a directory to S3
		*
		* @subcommand upload-directory
		* @synopsis <from> [<to>] [--sync] [--dry-run] [--concurrency=<concurrency>]
		* @synopsis <from> [<to>] [--concurrency=<concurrency>] [--verbose]
		*/
		public function upload_directory( $args, $args_assoc ) {

		@@ -310,26 +226,21 @@ class S3_Uploads_WP_CLI_Command extends WP_CLI_Command {
		}

		$s3 = S3_Uploads::get_instance()->s3();
		$bucket = strtok( S3_UPLOADS_BUCKET, '/' );
		$prefix = '';
		$args_assoc = wp_parse_args( $args_assoc, [ 'concurrency' => 5, 'verbose' => false ] );

		if ( strpos( S3_UPLOADS_BUCKET, '/' ) ) {
		$prefix = trailingslashit( str_replace( strtok( S3_UPLOADS_BUCKET, '/' ) . '/', '', S3_UPLOADS_BUCKET ) );
		$transfer_args = [
		'concurrency' => $args_assoc['concurrency'],
		'debug' => (bool) $args_assoc['verbose'],
		'before' => function ( AWS\Command $command ) {
		if ( in_array( $command->getName(), [ 'PutObject', 'CreateMultipartUpload' ], true ) ) {
		$acl = defined( 'S3_UPLOADS_OBJECT_ACL' ) ? S3_UPLOADS_OBJECT_ACL : 'public-read';
		$command['ACL'] = $acl;
		}

		},
		];
		try {
		$s3->uploadDirectory(
		$from,
		$bucket,
		$prefix . $to,
		array(
		'debug' => true,
		'params' => array( 'ACL' => 'public-read' ),
		'builder' => new S3_Uploads_UploadSyncBuilder( ! empty( $args_assoc['dry-run'] ) ),
		'force' => empty( $args_assoc['sync'] ),
		'concurrency' => ! empty( $args_assoc['concurrency'] ) ? $args_assoc['concurrency'] : 5,
		)
		);
		$manager = new Transfer( $s3, $from, 's3://' . S3_UPLOADS_BUCKET . '/' . $to, $transfer_args );
		$manager->transfer();
		} catch ( Exception $e ) {
		WP_CLI::error( $e->getMessage() );
		}