From c96a12a5cc1a1cee2c19bcaea4e3b7abb20966dc Mon Sep 17 00:00:00 2001 From: Viktor Vlasenko Date: Tue, 20 Oct 2020 14:06:17 +0000 Subject: [PATCH] CS-5747: create gridfs-config bundle - implemented business logic --- LICENSE | 23 + README.md | 5 + composer.json | 31 + .../GridFSConfigBundle/Adapter/GridFS.php | 117 +++ .../Factory/GridFSAdapterFactory.php | 57 ++ .../OroGridFSConfigExtension.php | 104 +++ .../GridFSConfigBundle/GridFS/Bucket.php | 764 ++++++++++++++++++ .../GridFS/CollectionWrapper.php | 362 +++++++++ .../GridFS/ReadableStream.php | 314 +++++++ .../GridFS/StreamWrapper.php | 346 ++++++++ .../GridFS/WritableStream.php | 301 +++++++ .../OroGridFSConfigBundle.php | 12 + .../Resources/config/adapter_factories.xml | 11 + .../Resources/config/oro/bundles.yml | 2 + .../Resources/config/oro_gridfs.yml | 4 + .../Resources/config/services.yml | 13 + .../Tests/Unit/Adapter/GridFSTest.php | 171 ++++ .../Tests/Unit/Adapter/test.txt | 1 + .../Factory/GridFSAdapterFactoryTest.php | 62 ++ .../OroGridFSConfigExtensionTest.php | 102 +++ 20 files changed, 2802 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 composer.json create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Adapter/GridFS.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/Factory/GridFSAdapterFactory.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/OroGridFSConfigExtension.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/GridFS/Bucket.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/GridFS/CollectionWrapper.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/GridFS/ReadableStream.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/GridFS/StreamWrapper.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/GridFS/WritableStream.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/OroGridFSConfigBundle.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Resources/config/adapter_factories.xml create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro/bundles.yml create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro_gridfs.yml create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Resources/config/services.yml create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/GridFSTest.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/test.txt create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/Factory/GridFSAdapterFactoryTest.php create mode 100644 src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/OroGridFSConfigExtensionTest.php diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..43a68e1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +OroPlatform + +The MIT License (MIT) + +Copyright (c) 2013, Oro, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..50b546e --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# OroGridFsConfigBundle + +OroGridFsConfigBundle provides configuration enhancements for Oro applications to enable usage of [GridFS Documentation](https://docs.mongodb.com/manual/core/gridfs/) as filesystem. + +Please see [documentation](https://doc.oroinc.com/master/backend/bundles/platform/GridFSConfigBundle/) for more details. diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..ce1db6a --- /dev/null +++ b/composer.json @@ -0,0 +1,31 @@ +{ + "name": "oro/gridfs-config", + "description": "GridFS adapter configuration for Gaufrette", + "type": "symfony-bundle", + "keywords": ["Oro", "OroPlatform", "Mongo", "GridFs"], + "homepage": "https://github.com/oroinc/gridfs-config", + "license": "MIT", + "autoload": { + "psr-4": {"Oro\\Bundle\\GridFSConfigBundle\\": "src/Oro/Bundle/GridFSConfigBundle"}, + "exclude-from-classmap": ["/Tests/"] + }, + "authors": [ + { + "name": "Oro, Inc", + "homepage": "http://oroinc.com" + } + ], + "require": { + "oro/platform": "3.1.*", + "ext-mongodb": "*", + "mongodb/mongodb": "^1.1", + "knplabs/knp-gaufrette-bundle": "0.3.*" + }, + "minimum-stability": "dev", + "prefer-stable": true, + "extra": { + "branch-alias": { + "dev-master": "3.1-dev" + } + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/Adapter/GridFS.php b/src/Oro/Bundle/GridFSConfigBundle/Adapter/GridFS.php new file mode 100644 index 0000000..6c90a5a --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Adapter/GridFS.php @@ -0,0 +1,117 @@ +bucket = $bucket; + } + + /** + * {@inheritdoc} + */ + public function read($key) + { + return parent::read($this->formatKey($key)); + } + + /** + * {@inheritdoc} + */ + public function write($key, $content) + { + if (empty($content)) { + return 0; + } + + // remove old file with same filename if one exist + $this->delete($key); + + $stream = $this->getBucket()->openUploadStream( + $this->formatKey($key), + ['contentType' => $this->guessContentType($content)] + ); + + try { + return fwrite($stream, $content); + } finally { + fclose($stream); + } + + return false; + } + + /** + * {@inheritDoc} + */ + public function exists($key) + { + return parent::exists($this->formatKey($key)); + } + + /** + * {@inheritDoc} + */ + public function delete($key) + { + return parent::delete($this->formatKey($key)); + } + + /** + * {@inheritDoc} + */ + public function rename($sourceKey, $targetKey) + { + return parent::rename($this->formatKey($sourceKey), $this->formatKey($targetKey)); + } + + /** + * @return Bucket + */ + public function getBucket(): Bucket + { + return $this->bucket; + } + + /** + * @param string $content + * + * @return string + */ + protected function guessContentType(string $content): string + { + $fileInfo = new \finfo(FILEINFO_MIME_TYPE); + + return $fileInfo->buffer($content); + } + + /** + * @param string $key + * + * @return string + */ + private function formatKey(string $key): string + { + return ltrim($key, '/'); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/Factory/GridFSAdapterFactory.php b/src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/Factory/GridFSAdapterFactory.php new file mode 100644 index 0000000..3ea56dd --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/Factory/GridFSAdapterFactory.php @@ -0,0 +1,57 @@ +\w+)$|', $dbConfig, $matches); + + $driverManagerDefinition = new ChildDefinition('oro.mongodb.driver.manager'); + $driverManagerDefinition->addArgument($dbConfig); + $container->setDefinition('oro.mongodb.driver.manager.' . $id, $driverManagerDefinition); + + $bucketDefinition = new ChildDefinition('oro.gridfs.bucket'); + $bucketDefinition->addArgument($driverManagerDefinition); + $bucketDefinition->addArgument($matches['db']); + $container->setDefinition('oro.gridfs.bucket.' . $id, $bucketDefinition); + + $adapterDefinition = new ChildDefinition('oro_gridfs.adapter.gridfs'); + $adapterDefinition->addArgument($bucketDefinition); + $container->setDefinition($id, $adapterDefinition); + } + + /** + * {@inheritdoc} + */ + public function getKey() + { + return 'oro_gridfs'; + } + + /** + * {@inheritdoc} + */ + public function addConfiguration(NodeDefinition $node) + { + $node + ->children() + ->scalarNode(self::DSN_STRING_PARAMETER)->isRequired()->cannotBeEmpty()->end() + ->end(); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/OroGridFSConfigExtension.php b/src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/OroGridFSConfigExtension.php new file mode 100644 index 0000000..2e10466 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/DependencyInjection/OroGridFSConfigExtension.php @@ -0,0 +1,104 @@ +load('services.yml'); + } + + /** + * @{inheritdoc} + */ + public function prepend(ContainerBuilder $container) + { + // register oro_gridfs gaufrette adapter + $container->setParameter('oro_gridfs.config_dir', __DIR__.'/../Resources/config'); + $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); + $loader->load('oro_gridfs.yml'); + + //process configuration in mongodb_gridfs_dsn_* parameters + $this->processContainerParameters($container); + } + + /** + * @param ContainerBuilder $container + */ + private function processContainerParameters(ContainerBuilder $container): void + { + $parametergrConfigs = $this->getGridfsDsnConfigs($container); + if (count($parametergrConfigs) === 0) { + return; + } + + $gaufretteConfigs = $container->getExtensionConfig('knp_gaufrette'); + $adapterNames = $this->collectConfiguredGaufretteAdaptersNames($gaufretteConfigs); + + $config = []; + $prefixLength = strlen(self::PARAM_MONGODB_DSN_PREFIX); + foreach ($parametergrConfigs as $gridfsDsnConfigKey => $dsnString) { + $adapterName = substr($gridfsDsnConfigKey, $prefixLength); + if (!in_array($adapterName, $adapterNames)) { + throw new \RuntimeException( + sprintf('Wrong Gaufrette DSN configuration. "%s" adapter cannot be found', $adapterName) + ); + } + $config[$adapterName] = [ + 'oro_gridfs' => [GridFSAdapterFactory::DSN_STRING_PARAMETER => $dsnString], + ]; + } + + $gaufretteConfigs[] = ['adapters' => $config]; + $container->setExtensionConfig('knp_gaufrette', $gaufretteConfigs); + } + + /** + * @param ContainerBuilder $container + * + * @return array + */ + private function getGridfsDsnConfigs(ContainerBuilder $container): array + { + $parametersConfigs = $container->getParameterBag()->all(); + return array_filter( + $parametersConfigs, + function ($key) { + return strpos($key, self::PARAM_MONGODB_DSN_PREFIX) === 0; + }, + ARRAY_FILTER_USE_KEY + ); + } + + /** + * @param array $gaufretteConfigs + * + * @return array + */ + private function collectConfiguredGaufretteAdaptersNames(array $gaufretteConfigs): array + { + $adapters = []; + foreach ($gaufretteConfigs as $config) { + if (empty($config['adapters'])) { + continue; + } + $adapters[] = array_keys($config['adapters']); + } + + return array_unique(array_merge(...$adapters)); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/GridFS/Bucket.php b/src/Oro/Bundle/GridFSConfigBundle/GridFS/Bucket.php new file mode 100644 index 0000000..1f3898d --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/GridFS/Bucket.php @@ -0,0 +1,764 @@ + BSONArray::class, + 'document' => BSONDocument::class, + 'root' => BSONDocument::class, + ]; + + /** @var string */ + private static $streamWrapperProtocol = 'gridfs'; + + /** @var CollectionWrapper */ + private $collectionWrapper; + + /** @var string */ + private $databaseName; + + /** @var Manager */ + private $manager; + + /** @var string */ + private $bucketName; + + /** @var boolean */ + private $disableMD5; + + /** @var integer */ + private $chunkSizeBytes; + + /** @var ReadConcern */ + private $readConcern; + + /** @var ReadPreference */ + private $readPreference; + + /** @var array */ + private $typeMap; + + /** @var WriteConcern */ + private $writeConcern; + + /** + * Constructs a GridFS bucket. + * + * Supported options: + * + * * bucketName (string): The bucket name, which will be used as a prefix + * for the files and chunks collections. Defaults to "fs". + * + * * chunkSizeBytes (integer): The chunk size in bytes. Defaults to + * 261120 (i.e. 255 KiB). + * + * * disableMD5 (boolean): When true, no MD5 sum will be generated for + * each stored file. Defaults to "false". + * + * * readConcern (MongoDB\Driver\ReadConcern): Read concern. + * + * * readPreference (MongoDB\Driver\ReadPreference): Read preference. + * + * * typeMap (array): Default type map for cursors and BSON documents. + * + * * writeConcern (MongoDB\Driver\WriteConcern): Write concern. + * + * @param Manager $manager Manager instance from the driver + * @param string $databaseName Database name + * @param array $options Bucket options + * + * @throws InvalidArgumentException for parameter/option parsing errors + * @SuppressWarnings(PHPMD.CyclomaticComplexity) + * @SuppressWarnings(PHPMD.NPathComplexity) + */ + public function __construct(Manager $manager, $databaseName, array $options = []) + { + $options += [ + 'bucketName' => self::$defaultBucketName, + 'chunkSizeBytes' => self::$defaultChunkSizeBytes, + 'disableMD5' => false, + ]; + + if (!is_string($options['bucketName'])) { + throw InvalidArgumentException::invalidType('"bucketName" option', $options['bucketName'], 'string'); + } + + if (!is_integer($options['chunkSizeBytes'])) { + throw InvalidArgumentException::invalidType( + '"chunkSizeBytes" option', + $options['chunkSizeBytes'], + 'integer' + ); + } + + if ($options['chunkSizeBytes'] < 1) { + throw new InvalidArgumentException( + sprintf('Expected "chunkSizeBytes" option to be >= 1, %d given', $options['chunkSizeBytes']) + ); + } + + if (!is_bool($options['disableMD5'])) { + throw InvalidArgumentException::invalidType('"disableMD5" option', $options['disableMD5'], 'boolean'); + } + + if (isset($options['readConcern']) && !$options['readConcern'] instanceof ReadConcern) { + throw InvalidArgumentException::invalidType( + '"readConcern" option', + $options['readConcern'], + ReadConcern::class + ); + } + + if (isset($options['readPreference']) && !$options['readPreference'] instanceof ReadPreference) { + throw InvalidArgumentException::invalidType( + '"readPreference" option', + $options['readPreference'], + ReadPreference::class + ); + } + + if (isset($options['typeMap']) && !is_array($options['typeMap'])) { + throw InvalidArgumentException::invalidType('"typeMap" option', $options['typeMap'], 'array'); + } + + if (isset($options['writeConcern']) && !$options['writeConcern'] instanceof WriteConcern) { + throw InvalidArgumentException::invalidType( + '"writeConcern" option', + $options['writeConcern'], + WriteConcern::class + ); + } + + $this->manager = $manager; + $this->databaseName = (string)$databaseName; + $this->bucketName = $options['bucketName']; + $this->chunkSizeBytes = $options['chunkSizeBytes']; + $this->disableMD5 = $options['disableMD5']; + $this->readConcern = isset($options['readConcern']) + ? $options['readConcern'] + : $this->manager->getReadConcern() + ; + $this->readPreference = isset($options['readPreference']) + ? $options['readPreference'] + : $this->manager->getReadPreference(); + $this->typeMap = isset($options['typeMap']) ? $options['typeMap'] : self::$defaultTypeMap; + $this->writeConcern = isset($options['writeConcern']) + ? $options['writeConcern'] + : $this->manager->getWriteConcern(); + + $collectionOptions = array_intersect_key( + $options, + ['readConcern' => 1, 'readPreference' => 1, 'typeMap' => 1, 'writeConcern' => 1] + ); + + $this->collectionWrapper = new CollectionWrapper( + $manager, + $databaseName, + $options['bucketName'], + $collectionOptions + ); + $this->registerStreamWrapper(); + } + + /** + * Return internal properties for debugging purposes. + * + * @see http://php.net/manual/en/language.oop5.magic.php#language.oop5.magic.debuginfo + * @return array + */ + public function __debugInfo() + { + return [ + 'bucketName' => $this->bucketName, + 'databaseName' => $this->databaseName, + 'manager' => $this->manager, + 'chunkSizeBytes' => $this->chunkSizeBytes, + 'readConcern' => $this->readConcern, + 'readPreference' => $this->readPreference, + 'typeMap' => $this->typeMap, + 'writeConcern' => $this->writeConcern, + ]; + } + + /** + * Delete a file from the GridFS bucket. + * + * If the files collection document is not found, this method will still + * attempt to delete orphaned chunks. + * + * @param mixed $id File ID + * + * @throws FileNotFoundException if no file could be selected + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function delete($id) + { + $file = $this->collectionWrapper->findFileById($id); + $this->collectionWrapper->deleteFileAndChunksById($id); + + if ($file === null) { + throw FileNotFoundException::byId($id, $this->getFilesNamespace()); + } + } + + /** + * Writes the contents of a GridFS file to a writable stream. + * + * @param mixed $id File ID + * @param resource $destination Writable Stream + * + * @throws FileNotFoundException if no file could be selected + * @throws InvalidArgumentException if $destination is not a stream + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function downloadToStream($id, $destination) + { + if (!is_resource($destination) || get_resource_type($destination) != "stream") { + throw InvalidArgumentException::invalidType('$destination', $destination, 'resource'); + } + + stream_copy_to_stream($this->openDownloadStream($id), $destination); + } + + /** + * Writes the contents of a GridFS file, which is selected by name and + * revision, to a writable stream. + * + * Supported options: + * + * * revision (integer): Which revision (i.e. documents with the same + * filename and different uploadDate) of the file to retrieve. Defaults + * to -1 (i.e. the most recent revision). + * + * Revision numbers are defined as follows: + * + * * 0 = the original stored file + * * 1 = the first revision + * * 2 = the second revision + * * etc… + * * -2 = the second most recent revision + * * -1 = the most recent revision + * + * @param string $filename Filename + * @param resource $destination Writable Stream + * @param array $options Download options + * + * @throws FileNotFoundException if no file could be selected + * @throws InvalidArgumentException if $destination is not a stream + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function downloadToStreamByName($filename, $destination, array $options = []) + { + if (!is_resource($destination) || get_resource_type($destination) != "stream") { + throw InvalidArgumentException::invalidType('$destination', $destination, 'resource'); + } + + stream_copy_to_stream($this->openDownloadStreamByName($filename, $options), $destination); + } + + /** + * Drops the files and chunks collections associated with this GridFS + * bucket. + * + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function drop() + { + $this->collectionWrapper->dropCollections(); + } + + /** + * Finds documents from the GridFS bucket's files collection matching the + * query. + * + * @param array|object $filter Query by which to filter documents + * @param array $options Additional options + * + * @return Cursor + * @throws UnsupportedException if options are not supported by the selected server + * @throws InvalidArgumentException for parameter/option parsing errors + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + * @see Find::__construct() for supported options + */ + public function find($filter = [], array $options = []) + { + return $this->collectionWrapper->findFiles($filter, $options); + } + + /** + * Finds a single document from the GridFS bucket's files collection + * matching the query. + * + * @param array|object $filter Query by which to filter documents + * @param array $options Additional options + * + * @return array|object|null + * @throws UnsupportedException if options are not supported by the selected server + * @throws InvalidArgumentException for parameter/option parsing errors + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + * @see FindOne::__construct() for supported options + */ + public function findOne($filter = [], array $options = []) + { + return $this->collectionWrapper->findOneFile($filter, $options); + } + + /** + * Return the bucket name. + * + * @return string + */ + public function getBucketName() + { + return $this->bucketName; + } + + /** + * Return the chunks collection. + * + * @return Collection + */ + public function getChunksCollection() + { + return $this->collectionWrapper->getChunksCollection(); + } + + /** + * Return the chunk size in bytes. + * + * @return integer + */ + public function getChunkSizeBytes() + { + return $this->chunkSizeBytes; + } + + /** + * Return the database name. + * + * @return string + */ + public function getDatabaseName() + { + return $this->databaseName; + } + + /** + * Gets the file document of the GridFS file associated with a stream. + * + * @param resource $stream GridFS stream + * + * @return array|object + * @throws InvalidArgumentException if $stream is not a GridFS stream + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function getFileDocumentForStream($stream) + { + $file = $this->getRawFileDocumentForStream($stream); + + // Filter the raw document through the specified type map + return apply_type_map_to_document($file, $this->typeMap); + } + + /** + * Gets the file document's ID of the GridFS file associated with a stream. + * + * @param resource $stream GridFS stream + * + * @return mixed + * @throws CorruptFileException if the file "_id" field does not exist + * @throws InvalidArgumentException if $stream is not a GridFS stream + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function getFileIdForStream($stream) + { + $file = $this->getRawFileDocumentForStream($stream); + + /* Filter the raw document through the specified type map, but override + * the root type so we can reliably access the ID. + */ + $typeMap = ['root' => 'stdClass'] + $this->typeMap; + $file = apply_type_map_to_document($file, $typeMap); + + if (!isset($file->_id) && !property_exists($file, '_id')) { + throw new CorruptFileException('file._id does not exist'); + } + + return $file->_id; + } + + /** + * Return the files collection. + * + * @return Collection + */ + public function getFilesCollection() + { + return $this->collectionWrapper->getFilesCollection(); + } + + /** + * Return the read concern for this GridFS bucket. + * + * @see http://php.net/manual/en/mongodb-driver-readconcern.isdefault.php + * @return ReadConcern + */ + public function getReadConcern() + { + return $this->readConcern; + } + + /** + * Return the read preference for this GridFS bucket. + * + * @return ReadPreference + */ + public function getReadPreference() + { + return $this->readPreference; + } + + /** + * Return the type map for this GridFS bucket. + * + * @return array + */ + public function getTypeMap() + { + return $this->typeMap; + } + + /** + * Return the write concern for this GridFS bucket. + * + * @see http://php.net/manual/en/mongodb-driver-writeconcern.isdefault.php + * @return WriteConcern + */ + public function getWriteConcern() + { + return $this->writeConcern; + } + + /** + * Opens a readable stream for reading a GridFS file. + * + * @param mixed $id File ID + * + * @return resource + * @throws FileNotFoundException if no file could be selected + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function openDownloadStream($id) + { + $file = $this->collectionWrapper->findFileById($id); + + if ($file === null) { + throw FileNotFoundException::byId($id, $this->getFilesNamespace()); + } + + return $this->openDownloadStreamByFile($file); + } + + /** + * Opens a readable stream stream to read a GridFS file, which is selected + * by name and revision. + * + * Supported options: + * + * * revision (integer): Which revision (i.e. documents with the same + * filename and different uploadDate) of the file to retrieve. Defaults + * to -1 (i.e. the most recent revision). + * + * Revision numbers are defined as follows: + * + * * 0 = the original stored file + * * 1 = the first revision + * * 2 = the second revision + * * etc… + * * -2 = the second most recent revision + * * -1 = the most recent revision + * + * @param string $filename Filename + * @param array $options Download options + * + * @return resource + * @throws FileNotFoundException if no file could be selected + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function openDownloadStreamByName($filename, array $options = []) + { + $options += ['revision' => -1]; + + $file = $this->collectionWrapper->findFileByFilenameAndRevision($filename, $options['revision']); + + if ($file === null) { + throw FileNotFoundException::byFilenameAndRevision( + $filename, + $options['revision'], + $this->getFilesNamespace() + ); + } + + return $this->openDownloadStreamByFile($file); + } + + /** + * Opens a writable stream for writing a GridFS file. + * + * Supported options: + * + * * _id (mixed): File document identifier. Defaults to a new ObjectId. + * + * * chunkSizeBytes (integer): The chunk size in bytes. Defaults to the + * bucket's chunk size. + * + * * disableMD5 (boolean): When true, no MD5 sum will be generated for + * the stored file. Defaults to "false". + * + * * metadata (document): User data for the "metadata" field of the files + * collection document. + * + * @param string $filename Filename + * @param array $options Upload options + * + * @return resource + */ + public function openUploadStream($filename, array $options = []) + { + $options += ['chunkSizeBytes' => $this->chunkSizeBytes]; + + $path = $this->createPathForUpload(); + $context = stream_context_create( + [ + self::$streamWrapperProtocol => [ + 'collectionWrapper' => $this->collectionWrapper, + 'filename' => $filename, + 'options' => $options, + ], + ] + ); + + return fopen($path, 'w', false, $context); + } + + /** + * Renames the GridFS file with the specified ID. + * + * @param mixed $id File ID + * @param string $newFilename New filename + * + * @throws FileNotFoundException if no file could be selected + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function rename($id, $newFilename) + { + $updateResult = $this->collectionWrapper->updateFilenameForId($id, $newFilename); + + if ($updateResult->getModifiedCount() === 1) { + return; + } + + /* If the update resulted in no modification, it's possible that the + * file did not exist, in which case we must raise an error. Checking + * the write result's matched count will be most efficient, but fall + * back to a findOne operation if necessary (i.e. legacy writes). + */ + $found = $updateResult->getMatchedCount() !== null + ? $updateResult->getMatchedCount() === 1 + : $this->collectionWrapper->findFileById($id) !== null; + + if (!$found) { + throw FileNotFoundException::byId($id, $this->getFilesNamespace()); + } + } + + /** + * Writes the contents of a readable stream to a GridFS file. + * + * Supported options: + * + * * _id (mixed): File document identifier. Defaults to a new ObjectId. + * + * * chunkSizeBytes (integer): The chunk size in bytes. Defaults to the + * bucket's chunk size. + * + * * disableMD5 (boolean): When true, no MD5 sum will be generated for + * the stored file. Defaults to "false". + * + * * metadata (document): User data for the "metadata" field of the files + * collection document. + * + * @param string $filename Filename + * @param resource $source Readable stream + * @param array $options Stream options + * + * @return mixed ID of the newly created GridFS file + * @throws InvalidArgumentException if $source is not a GridFS stream + * @throws DriverRuntimeException for other driver errors (e.g. connection errors) + */ + public function uploadFromStream($filename, $source, array $options = []) + { + if (!is_resource($source) || get_resource_type($source) != "stream") { + throw InvalidArgumentException::invalidType('$source', $source, 'resource'); + } + + $destination = $this->openUploadStream($filename, $options); + stream_copy_to_stream($source, $destination); + + return $this->getFileIdForStream($destination); + } + + /** + * Creates a path for an existing GridFS file. + * + * @param stdClass $file GridFS file document + * + * @return string + */ + private function createPathForFile(stdClass $file) + { + if (!is_object($file->_id) || method_exists($file->_id, '__toString')) { + $id = (string)$file->_id; + } else { + $id = toJSON(fromPHP(['_id' => $file->_id])); + } + + return sprintf( + '%s://%s/%s.files/%s', + self::$streamWrapperProtocol, + urlencode($this->databaseName), + urlencode($this->bucketName), + urlencode($id) + ); + } + + /** + * Creates a path for a new GridFS file, which does not yet have an ID. + * + * @return string + */ + private function createPathForUpload() + { + return sprintf( + '%s://%s/%s.files', + self::$streamWrapperProtocol, + urlencode($this->databaseName), + urlencode($this->bucketName) + ); + } + + /** + * Returns the names of the files collection. + * + * @return string + */ + private function getFilesNamespace() + { + return sprintf('%s.%s.files', $this->databaseName, $this->bucketName); + } + + /** + * Gets the file document of the GridFS file associated with a stream. + * + * This returns the raw document from the StreamWrapper, which does not + * respect the Bucket's type map. + * + * @param resource $stream GridFS stream + * + * @return stdClass + * @throws InvalidArgumentException + */ + private function getRawFileDocumentForStream($stream) + { + if (!is_resource($stream) || get_resource_type($stream) != "stream") { + throw InvalidArgumentException::invalidType('$stream', $stream, 'resource'); + } + + $metadata = stream_get_meta_data($stream); + + if (!isset($metadata['wrapper_data']) || !$metadata['wrapper_data'] instanceof StreamWrapper) { + throw InvalidArgumentException::invalidType( + '$stream wrapper data', + isset($metadata['wrapper_data']) ? $metadata['wrapper_data'] : null, + StreamWrapper::class + ); + } + + return $metadata['wrapper_data']->getFile(); + } + + /** + * Opens a readable stream for the GridFS file. + * + * @param stdClass $file GridFS file document + * + * @return resource + */ + private function openDownloadStreamByFile(stdClass $file) + { + $path = $this->createPathForFile($file); + $context = stream_context_create( + [ + self::$streamWrapperProtocol => [ + 'collectionWrapper' => $this->collectionWrapper, + 'file' => $file, + ], + ] + ); + + return fopen($path, 'r', false, $context); + } + + /** + * Registers the GridFS stream wrapper if it is not already registered. + */ + private function registerStreamWrapper() + { + if (in_array(self::$streamWrapperProtocol, stream_get_wrappers())) { + return; + } + + StreamWrapper::register(self::$streamWrapperProtocol); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/GridFS/CollectionWrapper.php b/src/Oro/Bundle/GridFSConfigBundle/GridFS/CollectionWrapper.php new file mode 100644 index 0000000..b73b2e9 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/GridFS/CollectionWrapper.php @@ -0,0 +1,362 @@ +databaseName = (string)$databaseName; + $this->bucketName = (string)$bucketName; + + $this->filesCollection = new Collection( + $manager, + $databaseName, + sprintf('%s.files', $bucketName), + $collectionOptions + ); + $this->chunksCollection = new Collection( + $manager, + $databaseName, + sprintf('%s.chunks', $bucketName), + $collectionOptions + ); + } + + /** + * Deletes all GridFS chunks for a given file ID. + * + * @param mixed $id + */ + public function deleteChunksByFilesId($id) + { + $this->chunksCollection->deleteMany(['files_id' => $id]); + } + + /** + * Deletes a GridFS file and related chunks by ID. + * + * @param mixed $id + */ + public function deleteFileAndChunksById($id) + { + $this->filesCollection->deleteOne(['_id' => $id]); + $this->chunksCollection->deleteMany(['files_id' => $id]); + } + + /** + * Drops the GridFS files and chunks collections. + */ + public function dropCollections() + { + $this->filesCollection->drop(['typeMap' => []]); + $this->chunksCollection->drop(['typeMap' => []]); + } + + /** + * Finds GridFS chunk documents for a given file ID and optional offset. + * + * @param mixed $id File ID + * @param integer $fromChunk Starting chunk (inclusive) + * + * @return Cursor + */ + public function findChunksByFileId($id, $fromChunk = 0) + { + return $this->chunksCollection->find( + [ + 'files_id' => $id, + 'n' => ['$gte' => $fromChunk], + ], + [ + 'sort' => ['n' => 1], + 'typeMap' => ['root' => 'stdClass'], + ] + ); + } + + /** + * Finds a GridFS file document for a given filename and revision. + * + * Revision numbers are defined as follows: + * + * * 0 = the original stored file + * * 1 = the first revision + * * 2 = the second revision + * * etc… + * * -2 = the second most recent revision + * * -1 = the most recent revision + * + * @param string $filename + * @param integer $revision + * + * @return stdClass|null + * @see Bucket::openDownloadStreamByName() + * @see Bucket::downloadToStreamByName() + */ + public function findFileByFilenameAndRevision($filename, $revision) + { + $filename = (string)$filename; + $revision = (integer)$revision; + + if ($revision < 0) { + $skip = abs($revision) - 1; + $sortOrder = -1; + } else { + $skip = $revision; + $sortOrder = 1; + } + + return $this->filesCollection->findOne( + ['filename' => $filename], + [ + 'skip' => $skip, + 'sort' => ['uploadDate' => $sortOrder], + 'typeMap' => ['root' => 'stdClass'], + ] + ); + } + + /** + * Finds a GridFS file document for a given ID. + * + * @param mixed $id + * + * @return stdClass|null + */ + public function findFileById($id) + { + return $this->filesCollection->findOne( + ['_id' => $id], + ['typeMap' => ['root' => 'stdClass']] + ); + } + + /** + * Finds documents from the GridFS bucket's files collection. + * + * @param array|object $filter Query by which to filter documents + * @param array $options Additional options + * + * @return Cursor + * @see Find::__construct() for supported options + */ + public function findFiles($filter, array $options = []) + { + return $this->filesCollection->find($filter, $options); + } + + /** + * Finds a single document from the GridFS bucket's files collection. + * + * @param array|object $filter Query by which to filter documents + * @param array $options Additional options + * + * @return array|object|null + */ + public function findOneFile($filter, array $options = []) + { + return $this->filesCollection->findOne($filter, $options); + } + + /** + * Return the bucket name. + * + * @return string + */ + public function getBucketName() + { + return $this->bucketName; + } + + /** + * Return the chunks collection. + * + * @return Collection + */ + public function getChunksCollection() + { + return $this->chunksCollection; + } + + /** + * Return the database name. + * + * @return string + */ + public function getDatabaseName() + { + return $this->databaseName; + } + + /** + * Return the files collection. + * + * @return Collection + */ + public function getFilesCollection() + { + return $this->filesCollection; + } + + /** + * Inserts a document into the chunks collection. + * + * @param array|object $chunk Chunk document + */ + public function insertChunk($chunk) + { + if (!$this->checkedIndexes) { + $this->ensureIndexes(); + } + + $this->chunksCollection->insertOne($chunk); + } + + /** + * Inserts a document into the files collection. + * + * The file document should be inserted after all chunks have been inserted. + * + * @param array|object $file File document + */ + public function insertFile($file) + { + if (!$this->checkedIndexes) { + $this->ensureIndexes(); + } + + $this->filesCollection->insertOne($file); + } + + /** + * Updates the filename field in the file document for a given ID. + * + * @param mixed $id + * @param string $filename + * + * @return UpdateResult + */ + public function updateFilenameForId($id, $filename) + { + return $this->filesCollection->updateOne( + ['_id' => $id], + ['$set' => ['filename' => (string)$filename]] + ); + } + + /** + * Create an index on the chunks collection if it does not already exist. + */ + private function ensureChunksIndex() + { + foreach ($this->chunksCollection->listIndexes() as $index) { + if ($index->isUnique() && $index->getKey() === ['files_id' => 1, 'n' => 1]) { + return; + } + } + + $this->chunksCollection->createIndex(['files_id' => 1, 'n' => 1], ['unique' => true]); + } + + /** + * Create an index on the files collection if it does not already exist. + */ + private function ensureFilesIndex() + { + foreach ($this->filesCollection->listIndexes() as $index) { + if ($index->isUnique() && $index->getKey() === ['filename' => 1]) { + return; + } + } + $this->filesCollection->createIndex(['filename' => 1], ['unique' => true]); + } + + /** + * Ensure indexes on the files and chunks collections exist. + * + * This method is called once before the first write operation on a GridFS + * bucket. Indexes are only be created if the files collection is empty. + */ + private function ensureIndexes() + { + if ($this->checkedIndexes) { + return; + } + + $this->checkedIndexes = true; + + if (!$this->isFilesCollectionEmpty()) { + return; + } + + $this->ensureFilesIndex(); + $this->ensureChunksIndex(); + } + + /** + * Returns whether the files collection is empty. + * + * @return boolean + */ + private function isFilesCollectionEmpty() + { + return null === $this->filesCollection->findOne( + [], + [ + 'readPreference' => new ReadPreference(ReadPreference::RP_PRIMARY), + 'projection' => ['_id' => 1], + 'typeMap' => [], + ] + ); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/GridFS/ReadableStream.php b/src/Oro/Bundle/GridFSConfigBundle/GridFS/ReadableStream.php new file mode 100644 index 0000000..a2b0a28 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/GridFS/ReadableStream.php @@ -0,0 +1,314 @@ +chunkSize) || !is_integer($file->chunkSize) || $file->chunkSize < 1) { + throw new CorruptFileException('file.chunkSize is not an integer >= 1'); + } + + if (!isset($file->length) || !is_integer($file->length) || $file->length < 0) { + throw new CorruptFileException('file.length is not an integer > 0'); + } + + if (!isset($file->_id) && !property_exists($file, '_id')) { + throw new CorruptFileException('file._id does not exist'); + } + + $this->file = $file; + $this->chunkSize = (integer)$file->chunkSize; + $this->length = (integer)$file->length; + + $this->collectionWrapper = $collectionWrapper; + + if ($this->length > 0) { + $this->numChunks = (integer)ceil($this->length / $this->chunkSize); + $this->expectedLastChunkSize = ($this->length - (($this->numChunks - 1) * $this->chunkSize)); + } + } + + /** + * Return internal properties for debugging purposes. + * + * @see http://php.net/manual/en/language.oop5.magic.php#language.oop5.magic.debuginfo + * @return array + */ + public function __debugInfo() + { + return [ + 'bucketName' => $this->collectionWrapper->getBucketName(), + 'databaseName' => $this->collectionWrapper->getDatabaseName(), + 'file' => $this->file, + ]; + } + + public function close() + { + // Nothing to do + } + + /** + * Return the stream's file document. + * + * @return stdClass + */ + public function getFile() + { + return $this->file; + } + + /** + * Return the stream's size in bytes. + * + * @return integer + */ + public function getSize() + { + return $this->length; + } + + /** + * Return whether the current read position is at the end of the stream. + * + * @return boolean + */ + public function isEOF() + { + if ($this->chunkOffset === $this->numChunks - 1) { + return $this->bufferOffset >= $this->expectedLastChunkSize; + } + + return $this->chunkOffset >= $this->numChunks; + } + + /** + * Read bytes from the stream. + * + * Note: this method may return a string smaller than the requested length + * if data is not available to be read. + * + * @param integer $length Number of bytes to read + * + * @return string + * @throws InvalidArgumentException if $length is negative + */ + public function readBytes($length) + { + if ($length < 0) { + throw new InvalidArgumentException(sprintf('$length must be >= 0; given: %d', $length)); + } + + if ($this->chunksIterator === null) { + $this->initChunksIterator(); + } + + if ($this->buffer === null && !$this->initBufferFromCurrentChunk()) { + return ''; + } + + $data = ''; + + while (strlen($data) < $length) { + if ($this->bufferOffset >= strlen($this->buffer) && !$this->initBufferFromNextChunk()) { + break; + } + + $initialDataLength = strlen($data); + $data .= substr($this->buffer, $this->bufferOffset, $length - $initialDataLength); + $this->bufferOffset += strlen($data) - $initialDataLength; + } + + return $data; + } + + /** + * Seeks the chunk and buffer offsets for the next read operation. + * + * @param integer $offset + * + * @throws InvalidArgumentException if $offset is out of range + */ + public function seek($offset) + { + if ($offset < 0 || $offset > $this->file->length) { + throw new InvalidArgumentException( + sprintf('$offset must be >= 0 and <= %d; given: %d', $this->file->length, $offset) + ); + } + + /* Compute the offsets for the chunk and buffer (i.e. chunk data) from + * which we will expect to read after seeking. If the chunk offset + * changed, we'll also need to reset the buffer. + */ + $lastChunkOffset = $this->chunkOffset; + $this->chunkOffset = (integer)floor($offset / $this->chunkSize); + $this->bufferOffset = $offset % $this->chunkSize; + + if ($lastChunkOffset === $this->chunkOffset) { + return; + } + + if ($this->chunksIterator === null) { + return; + } + + // Clear the buffer since the current chunk will be changed + $this->buffer = null; + + /* If we are seeking to a previous chunk, we need to reinitialize the + * chunk iterator. + */ + if ($lastChunkOffset > $this->chunkOffset) { + $this->chunksIterator = null; + + return; + } + + /* If we are seeking to a subsequent chunk, we do not need to + * reinitalize the chunk iterator. Instead, we can simply move forward + * to $this->chunkOffset. + */ + $numChunks = $this->chunkOffset - $lastChunkOffset; + for ($i = 0; $i < $numChunks; $i++) { + $this->chunksIterator->next(); + } + } + + /** + * Return the current position of the stream. + * + * This is the offset within the stream where the next byte would be read. + * + * @return integer + */ + public function tell() + { + return ($this->chunkOffset * $this->chunkSize) + $this->bufferOffset; + } + + /** + * Initialize the buffer to the current chunk's data. + * + * @return boolean Whether there was a current chunk to read + * @throws CorruptFileException if an expected chunk could not be read successfully + */ + private function initBufferFromCurrentChunk() + { + if ($this->chunkOffset === 0 && $this->numChunks === 0) { + return false; + } + + if (!$this->chunksIterator->valid()) { + throw CorruptFileException::missingChunk($this->chunkOffset); + } + + $currentChunk = $this->chunksIterator->current(); + + if ($currentChunk->n !== $this->chunkOffset) { + throw CorruptFileException::unexpectedIndex($currentChunk->n, $this->chunkOffset); + } + + $this->buffer = $currentChunk->data->getData(); + + $actualChunkSize = strlen($this->buffer); + + $expectedChunkSize = $this->chunkOffset === $this->numChunks - 1 + ? $this->expectedLastChunkSize + : $this->chunkSize; + + if ($actualChunkSize !== $expectedChunkSize) { + throw CorruptFileException::unexpectedSize($actualChunkSize, $expectedChunkSize); + } + + return true; + } + + /** + * Advance to the next chunk and initialize the buffer to its data. + * + * @return boolean Whether there was a next chunk to read + * @throws CorruptFileException if an expected chunk could not be read successfully + */ + private function initBufferFromNextChunk() + { + if ($this->chunkOffset === $this->numChunks - 1) { + return false; + } + + $this->bufferOffset = 0; + $this->chunkOffset++; + $this->chunksIterator->next(); + + return $this->initBufferFromCurrentChunk(); + } + + /** + * Initializes the chunk iterator starting from the current offset. + */ + private function initChunksIterator() + { + $cursor = $this->collectionWrapper->findChunksByFileId($this->file->_id, $this->chunkOffset); + + $this->chunksIterator = new IteratorIterator($cursor); + $this->chunksIterator->rewind(); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/GridFS/StreamWrapper.php b/src/Oro/Bundle/GridFSConfigBundle/GridFS/StreamWrapper.php new file mode 100644 index 0000000..8235817 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/GridFS/StreamWrapper.php @@ -0,0 +1,346 @@ +stream->getFile(); + } + + /** + * Register the GridFS stream wrapper. + * + * @param string $protocol Protocol to use for stream_wrapper_register() + */ + public static function register($protocol = 'gridfs') + { + if (in_array($protocol, stream_get_wrappers())) { + stream_wrapper_unregister($protocol); + } + + stream_wrapper_register($protocol, static::class, STREAM_IS_URL); + } + + /** + * Closes the stream. + * + * @see http://php.net/manual/en/streamwrapper.stream-close.php + */ + //@codingStandardsIgnoreStart + public function stream_close() + { + $this->stream->close(); + } + //@codingStandardsIgnoreEnd + + /** + * Returns whether the file pointer is at the end of the stream. + * + * @see http://php.net/manual/en/streamwrapper.stream-eof.php + * @return boolean + */ + //@codingStandardsIgnoreStart + public function stream_eof() + { + if (!$this->stream instanceof ReadableStream) { + return false; + } + + return $this->stream->isEOF(); + } + //@codingStandardsIgnoreEnd + + /** + * Opens the stream. + * + * @see http://php.net/manual/en/streamwrapper.stream-open.php + * + * @param string $path Path to the file resource + * @param string $mode Mode used to open the file (only "r" and "w" are supported) + * @param integer $options Additional flags set by the streams API + * @param string $openedPath Not used + * + * @return boolean + */ + //@codingStandardsIgnoreStart + public function stream_open($path, $mode, $options, &$openedPath) + { + $this->initProtocol($path); + $this->mode = $mode; + + if ($mode === 'r') { + return $this->initReadableStream(); + } + + if ($mode === 'w') { + return $this->initWritableStream(); + } + + return false; + } + //@codingStandardsIgnoreEnd + + /** + * Read bytes from the stream. + * + * Note: this method may return a string smaller than the requested length + * if data is not available to be read. + * + * @see http://php.net/manual/en/streamwrapper.stream-read.php + * + * @param integer $length Number of bytes to read + * + * @return string + */ + //@codingStandardsIgnoreStart + public function stream_read($length) + { + if (!$this->stream instanceof ReadableStream) { + return ''; + } + + try { + return $this->stream->readBytes($length); + } catch (Exception $e) { + trigger_error(sprintf('%s: %s', get_class($e), $e->getMessage()), E_USER_WARNING); + + return false; + } + } + //@codingStandardsIgnoreEnd + + /** + * Return the current position of the stream. + * + * @see http://php.net/manual/en/streamwrapper.stream-seek.php + * + * @param integer $offset Stream offset to seek to + * @param integer $whence One of SEEK_SET, SEEK_CUR, or SEEK_END + * + * @return boolean True if the position was updated and false otherwise + */ + //@codingStandardsIgnoreStart + public function stream_seek($offset, $whence = SEEK_SET) + { + $size = $this->stream->getSize(); + + if ($whence === SEEK_CUR) { + $offset += $this->stream->tell(); + } + + if ($whence === SEEK_END) { + $offset += $size; + } + + // WritableStreams are always positioned at the end of the stream + if ($this->stream instanceof WritableStream) { + return $offset === $size; + } + + if ($offset < 0 || $offset > $size) { + return false; + } + + $this->stream->seek($offset); + + return true; + } + //@codingStandardsIgnoreEnd + + /** + * Return information about the stream. + * + * @see http://php.net/manual/en/streamwrapper.stream-stat.php + * @return array + */ + //@codingStandardsIgnoreStart + public function stream_stat() + { + $stat = $this->getStatTemplate(); + + $stat[2] = $stat['mode'] = $this->stream instanceof ReadableStream + ? 0100444 // S_IFREG & S_IRUSR & S_IRGRP & S_IROTH + : 0100222; // S_IFREG & S_IWUSR & S_IWGRP & S_IWOTH + $stat[7] = $stat['size'] = $this->stream->getSize(); + + $file = $this->stream->getFile(); + + if (isset($file->uploadDate) && $file->uploadDate instanceof UTCDateTime) { + $timestamp = $file->uploadDate->toDateTime()->getTimestamp(); + $stat[9] = $stat['mtime'] = $timestamp; + $stat[10] = $stat['ctime'] = $timestamp; + } + + if (isset($file->chunkSize) && is_integer($file->chunkSize)) { + $stat[11] = $stat['blksize'] = $file->chunkSize; + } + + return $stat; + } + //@codingStandardsIgnoreEnd + + /** + * Return the current position of the stream. + * + * @see http://php.net/manual/en/streamwrapper.stream-tell.php + * @return integer The current position of the stream + */ + //@codingStandardsIgnoreStart + public function stream_tell() + { + return $this->stream->tell(); + } + //@codingStandardsIgnoreEnd + + /** + * Write bytes to the stream. + * + * @see http://php.net/manual/en/streamwrapper.stream-write.php + * + * @param string $data Data to write + * + * @return integer The number of bytes written + */ + //@codingStandardsIgnoreStart + public function stream_write($data) + { + if (!$this->stream instanceof WritableStream) { + return 0; + } + + try { + return $this->stream->writeBytes($data); + } catch (Exception $e) { + trigger_error(sprintf('%s: %s', get_class($e), $e->getMessage()), E_USER_WARNING); + + return false; + } + } + //@codingStandardsIgnoreEnd + + /** + * Returns a stat template with default values. + * + * @return array + */ + private function getStatTemplate() + { + return [ + // phpcs:disable Squiz.Arrays.ArrayDeclaration.IndexNoNewline + 0 => 0, + 'dev' => 0, + 1 => 0, + 'ino' => 0, + 2 => 0, + 'mode' => 0, + 3 => 0, + 'nlink' => 0, + 4 => 0, + 'uid' => 0, + 5 => 0, + 'gid' => 0, + 6 => -1, + 'rdev' => -1, + 7 => 0, + 'size' => 0, + 8 => 0, + 'atime' => 0, + 9 => 0, + 'mtime' => 0, + 10 => 0, + 'ctime' => 0, + 11 => -1, + 'blksize' => -1, + 12 => -1, + 'blocks' => -1, + // phpcs:enable + ]; + } + + /** + * Initialize the protocol from the given path. + * + * @param string $path + * + * @see StreamWrapper::stream_open() + */ + private function initProtocol($path) + { + $parts = explode('://', $path, 2); + $this->protocol = $parts[0] ?: 'gridfs'; + } + + /** + * Initialize the internal stream for reading. + * + * @return boolean + * @see StreamWrapper::stream_open() + */ + private function initReadableStream() + { + $context = stream_context_get_options($this->context); + + $this->stream = new ReadableStream( + $context[$this->protocol]['collectionWrapper'], + $context[$this->protocol]['file'] + ); + + return true; + } + + /** + * Initialize the internal stream for writing. + * + * @return boolean + * @see StreamWrapper::stream_open() + */ + private function initWritableStream() + { + $context = stream_context_get_options($this->context); + + $this->stream = new WritableStream( + $context[$this->protocol]['collectionWrapper'], + $context[$this->protocol]['filename'], + $context[$this->protocol]['options'] + ); + + return true; + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/GridFS/WritableStream.php b/src/Oro/Bundle/GridFSConfigBundle/GridFS/WritableStream.php new file mode 100644 index 0000000..43bca02 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/GridFS/WritableStream.php @@ -0,0 +1,301 @@ + new ObjectId(), + 'chunkSizeBytes' => self::$defaultChunkSizeBytes, + 'disableMD5' => false, + ]; + + if (isset($options['aliases']) && !is_string_array($options['aliases'])) { + throw InvalidArgumentException::invalidType('"aliases" option', $options['aliases'], 'array of strings'); + } + + if (!is_integer($options['chunkSizeBytes'])) { + throw InvalidArgumentException::invalidType( + '"chunkSizeBytes" option', + $options['chunkSizeBytes'], + 'integer' + ); + } + + if ($options['chunkSizeBytes'] < 1) { + throw new InvalidArgumentException( + sprintf('Expected "chunkSizeBytes" option to be >= 1, %d given', $options['chunkSizeBytes']) + ); + } + + if (!is_bool($options['disableMD5'])) { + throw InvalidArgumentException::invalidType('"disableMD5" option', $options['disableMD5'], 'boolean'); + } + + if (isset($options['contentType']) && !is_string($options['contentType'])) { + throw InvalidArgumentException::invalidType('"contentType" option', $options['contentType'], 'string'); + } + + if (isset($options['metadata']) && !is_array($options['metadata']) && !is_object($options['metadata'])) { + throw InvalidArgumentException::invalidType('"metadata" option', $options['metadata'], 'array or object'); + } + + $this->chunkSize = $options['chunkSizeBytes']; + $this->collectionWrapper = $collectionWrapper; + $this->disableMD5 = $options['disableMD5']; + + if (!$this->disableMD5) { + $this->hashCtx = hash_init('md5'); + } + + $this->file = [ + '_id' => $options['_id'], + 'chunkSize' => $this->chunkSize, + 'filename' => (string)$filename, + ] + array_intersect_key($options, ['aliases' => 1, 'contentType' => 1, 'metadata' => 1]); + } + + /** + * Return internal properties for debugging purposes. + * + * @see http://php.net/manual/en/language.oop5.magic.php#language.oop5.magic.debuginfo + * @return array + */ + public function __debugInfo() + { + return [ + 'bucketName' => $this->collectionWrapper->getBucketName(), + 'databaseName' => $this->collectionWrapper->getDatabaseName(), + 'file' => $this->file, + ]; + } + + /** + * Closes an active stream and flushes all buffered data to GridFS. + */ + public function close() + { + if ($this->isClosed) { + return; + } + + if (strlen($this->buffer) > 0) { + $this->insertChunkFromBuffer(); + } + + $this->fileCollectionInsert(); + $this->isClosed = true; + } + + /** + * Return the stream's file document. + * + * @return stdClass + */ + public function getFile() + { + return (object)$this->file; + } + + /** + * Return the stream's size in bytes. + * + * Note: this value will increase as more data is written to the stream. + * + * @return integer + */ + public function getSize() + { + return $this->length + strlen($this->buffer); + } + + /** + * Return the current position of the stream. + * + * This is the offset within the stream where the next byte would be + * written. Since seeking is not supported and writes are appended, this is + * always the end of the stream. + * + * @return integer + * @see WritableStream::getSize() + */ + public function tell() + { + return $this->getSize(); + } + + /** + * Inserts binary data into GridFS via chunks. + * + * Data will be buffered internally until chunkSizeBytes are accumulated, at + * which point a chunk document will be inserted and the buffer reset. + * + * @param string $data Binary data to write + * + * @return integer + */ + public function writeBytes($data) + { + if ($this->isClosed) { + return; + } + + $bytesRead = 0; + + while ($bytesRead != strlen($data)) { + $initialBufferLength = strlen($this->buffer); + $this->buffer .= substr($data, $bytesRead, $this->chunkSize - $initialBufferLength); + $bytesRead += strlen($this->buffer) - $initialBufferLength; + + if (strlen($this->buffer) == $this->chunkSize) { + $this->insertChunkFromBuffer(); + } + } + + return $bytesRead; + } + + private function abort() + { + try { + $this->collectionWrapper->deleteChunksByFilesId($this->file['_id']); + } catch (DriverRuntimeException $e) { + // We are already handling an error if abort() is called, so suppress this + } + + $this->isClosed = true; + } + + private function fileCollectionInsert() + { + $this->file['length'] = $this->length; + $this->file['uploadDate'] = new UTCDateTime(); + + if (!$this->disableMD5) { + $this->file['md5'] = hash_final($this->hashCtx); + } + + try { + $this->collectionWrapper->insertFile($this->file); + } catch (DriverRuntimeException $e) { + $this->abort(); + + throw $e; + } + + return $this->file['_id']; + } + + private function insertChunkFromBuffer() + { + if (strlen($this->buffer) == 0) { + return; + } + + $data = $this->buffer; + $this->buffer = ''; + + $chunk = [ + 'files_id' => $this->file['_id'], + 'n' => $this->chunkOffset, + 'data' => new Binary($data, Binary::TYPE_GENERIC), + ]; + + if (!$this->disableMD5) { + hash_update($this->hashCtx, $data); + } + + try { + $this->collectionWrapper->insertChunk($chunk); + } catch (DriverRuntimeException $e) { + $this->abort(); + + throw $e; + } + + $this->length += strlen($data); + $this->chunkOffset++; + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/OroGridFSConfigBundle.php b/src/Oro/Bundle/GridFSConfigBundle/OroGridFSConfigBundle.php new file mode 100644 index 0000000..6e2db1b --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/OroGridFSConfigBundle.php @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro/bundles.yml b/src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro/bundles.yml new file mode 100644 index 0000000..39a4028 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro/bundles.yml @@ -0,0 +1,2 @@ +bundles: + - { name: Oro\Bundle\GridFSConfigBundle\OroGridFSConfigBundle, priority: 200 } diff --git a/src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro_gridfs.yml b/src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro_gridfs.yml new file mode 100644 index 0000000..be2bf08 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Resources/config/oro_gridfs.yml @@ -0,0 +1,4 @@ +knp_gaufrette: + # @see https://github.com/KnpLabs/KnpGaufretteBundle/issues/121 + factories: + - "%oro_gridfs.config_dir%/adapter_factories.xml" diff --git a/src/Oro/Bundle/GridFSConfigBundle/Resources/config/services.yml b/src/Oro/Bundle/GridFSConfigBundle/Resources/config/services.yml new file mode 100644 index 0000000..223e753 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Resources/config/services.yml @@ -0,0 +1,13 @@ +services: + oro_gridfs.adapter.gridfs: + parent: knp_gaufrette.adapter.gridfs + class: Oro\Bundle\GridFSConfigBundle\Adapter\GridFS + abstract: true + + oro.mongodb.driver.manager: + class: MongoDB\Driver\Manager + abstract: true + + oro.gridfs.bucket: + class: Oro\Bundle\GridFSConfigBundle\GridFS\Bucket + abstract: true diff --git a/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/GridFSTest.php b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/GridFSTest.php new file mode 100644 index 0000000..1619dd4 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/GridFSTest.php @@ -0,0 +1,171 @@ +mongoDBBucketMock = $this->createMock(Bucket::class); + + $this->gridFSAdapter = new GridFS($this->mongoDBBucketMock); + } + + public function testWriteParentNotCalledWithEmptyContent() + { + $this->mongoDBBucketMock + ->expects(self::never()) + ->method('openUploadStream'); + + $this->gridFSAdapter->write('test', ''); + } + + public function testWriteOnExistingKey() + { + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('findOne') + ->with(['filename' => 'test.txt']) + ->willReturn(['_id' => '5f57c695ac49b642ae71f12c', 'data' => 'some data', 'filename' => 'test.txt']); + + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('delete') + ->with('5f57c695ac49b642ae71f12c'); + + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('openUploadStream') + ->with( + 'test.txt', + ['contentType' => 'text/plain'] + ) + ->willReturn(fopen('php://temp', 'w+b')); + + self::assertEquals(17, $this->gridFSAdapter->write('/test.txt', 'not empty content')); + } + + public function testWriteOnNonExistingKey() + { + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('findOne') + ->willReturn(null); + + $this->mongoDBBucketMock + ->expects(self::never()) + ->method('delete'); + + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('openUploadStream') + ->with( + 'test', + ['contentType' => 'text/plain'] + ) + ->willReturn(fopen('php://temp', 'w+b')); + + self::assertEquals(17, $this->gridFSAdapter->write('test', 'not empty content')); + } + + public function testTryToWriteOnExceptionDuringWrite() + { + $this->mongoDBBucketMock + ->expects(self::any()) + ->method('findOne') + ->with(['filename' => 'test']) + ->willReturn(null); + + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('openUploadStream') + ->with( + 'test', + ['contentType' => 'text/plain'] + ) + ->willReturn(fopen('php://temp', 'r')); + + self::assertFalse((bool)$this->gridFSAdapter->write('test', 'not empty content')); + } + + public function testRead() + { + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('openDownloadStreamByName') + ->with('test.txt') + ->willReturn(fopen(__DIR__ . '/test.txt', 'r')); + + $expectedContent = "some text\n"; + + self::assertEquals($expectedContent, $this->gridFSAdapter->read('/test.txt')); + } + + public function testExistsWithExistFile() + { + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('findOne') + ->with(['filename' => 'test.txt']) + ->willReturn(['_id' => '5f57c695ac49b642ae71f12c', 'data' => 'some data', 'filename' => 'test.txt']); + + self::assertTrue($this->gridFSAdapter->exists('/test.txt')); + } + + public function testExistsWithNonExistFile() + { + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('findOne') + ->with(['filename' => 'test.txt']) + ->willReturn(null); + + self::assertFalse($this->gridFSAdapter->exists('/test.txt')); + } + + public function testDeleteOnExistingFile() + { + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('findOne') + ->with(['filename' => 'test.txt']) + ->willReturn(['_id' => '5f57c695ac49b642ae71f12c', 'data' => 'some data', 'filename' => 'test.txt']); + + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('delete') + ->with('5f57c695ac49b642ae71f12c'); + + self::assertTrue($this->gridFSAdapter->delete('/test.txt')); + } + + public function testDeleteOnNonExistingFile() + { + $this->mongoDBBucketMock + ->expects(self::once()) + ->method('findOne') + ->with(['filename' => 'test.txt']) + ->willReturn(null); + + $this->mongoDBBucketMock + ->expects(self::never()) + ->method('delete'); + + self::assertFalse($this->gridFSAdapter->delete('/test.txt')); + } + + public function testGetBucket() + { + self::assertEquals($this->mongoDBBucketMock, $this->gridFSAdapter->getBucket()); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/test.txt b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/test.txt new file mode 100644 index 0000000..7b57bd2 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/Adapter/test.txt @@ -0,0 +1 @@ +some text diff --git a/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/Factory/GridFSAdapterFactoryTest.php b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/Factory/GridFSAdapterFactoryTest.php new file mode 100644 index 0000000..a5a0e76 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/Factory/GridFSAdapterFactoryTest.php @@ -0,0 +1,62 @@ +factory = new GridFSAdapterFactory(); + } + + public function testGetKey() + { + self::assertEquals('oro_gridfs', $this->factory->getKey()); + } + + public function testCreate() + { + $container = new ContainerBuilder(); + $id = 'test_gridfs.adapter'; + $config = ['mongodb_gridfs_dsn' => 'mongodb://user:password@host:27017/attachment']; + + $this->factory->create($container, $id, $config); + + self::assertTrue($container->hasDefinition('oro.mongodb.driver.manager.test_gridfs.adapter')); + self::assertTrue($container->hasDefinition('oro.gridfs.bucket.test_gridfs.adapter')); + self::assertTrue($container->hasDefinition('test_gridfs.adapter')); + + $bucketConfig = $container->getDefinition('oro.mongodb.driver.manager.test_gridfs.adapter'); + self::assertEquals('mongodb://user:password@host:27017/attachment', $bucketConfig->getArgument(0)); + + $bucketConfig = $container->getDefinition('oro.gridfs.bucket.test_gridfs.adapter'); + self::assertEquals('attachment', $bucketConfig->getArgument(1)); + } + + public function testCreateWithClusterConfiguration() + { + $container = new ContainerBuilder(); + $id = 'test_gridfs.adapter'; + $config = ['mongodb_gridfs_dsn' => 'mongodb://user:password@host1:27017,host2:27017/cache']; + + $this->factory->create($container, $id, $config); + + self::assertTrue($container->hasDefinition('oro.mongodb.driver.manager.test_gridfs.adapter')); + self::assertTrue($container->hasDefinition('oro.gridfs.bucket.test_gridfs.adapter')); + self::assertTrue($container->hasDefinition('test_gridfs.adapter')); + + $bucketConfig = $container->getDefinition('oro.mongodb.driver.manager.test_gridfs.adapter'); + self::assertEquals('mongodb://user:password@host1:27017,host2:27017/cache', $bucketConfig->getArgument(0)); + + $bucketConfig = $container->getDefinition('oro.gridfs.bucket.test_gridfs.adapter'); + self::assertEquals('cache', $bucketConfig->getArgument(1)); + } +} diff --git a/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/OroGridFSConfigExtensionTest.php b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/OroGridFSConfigExtensionTest.php new file mode 100644 index 0000000..56a7316 --- /dev/null +++ b/src/Oro/Bundle/GridFSConfigBundle/Tests/Unit/DependencyInjection/OroGridFSConfigExtensionTest.php @@ -0,0 +1,102 @@ +container = new ExtendedContainerBuilder(); + $this->container->registerExtension(new KnpGaufretteExtension()); + $this->container->setExtensionConfig( + 'knp_gaufrette', + [ + [ + 'adapters' => [ + 'first_adapter' => 'config1', + 'second_adapter' => 'config2', + ] + ] + ] + ); + + $this->extension = new OroGridFSConfigExtension(); + } + + public function testLoad() + { + $this->extension->load([], $this->container); + + self::assertTrue($this->container->has('oro_gridfs.adapter.gridfs')); + self::assertTrue($this->container->has('oro.mongodb.driver.manager')); + self::assertTrue($this->container->has('oro.gridfs.bucket')); + } + + public function testPrependWithoutGaufretteConfiguredParameters() + { + $this->extension->prepend($this->container); + + self::assertEquals( + [ + [ + 'adapters' => [ + 'first_adapter' => 'config1', + 'second_adapter' => 'config2', + ] + ], + [ + 'factories' => [ + '%oro_gridfs.config_dir%/adapter_factories.xml', + ] + ] + ], + $this->container->getExtensionConfig('knp_gaufrette') + ); + } + + public function testPrependOnReconfiguredFirstAdapter() + { + $this->container->setParameter('mongodb_gridfs_dsn_first_adapter', 'mongodb://user@host:27017/test'); + $this->extension->prepend($this->container); + + self::assertEquals( + [ + [ + 'adapters' => ['first_adapter' => 'config1', 'second_adapter' => 'config2'], + ], + [ + 'factories' => [ + '%oro_gridfs.config_dir%/adapter_factories.xml', + ] + ], + [ + 'adapters' => [ + 'first_adapter' => [ + 'oro_gridfs' => ['mongodb_gridfs_dsn' => 'mongodb://user@host:27017/test'] + ] + ] + ], + ], + $this->container->getExtensionConfig('knp_gaufrette') + ); + } + + public function testTryToPrependOnNonExistAdapter() + { + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Wrong Gaufrette DSN configuration. "not_existing" adapter cannot be found'); + + $this->container->setParameter('mongodb_gridfs_dsn_not_existing', 'mongodb://user@host:27017/test'); + $this->extension->prepend($this->container); + } +}