X Tutup
Skip to content

[For 10.4] Fix "files:transfer-ownership" in S3 multibucket setups #36464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
micbar merged 2 commits into from
Dec 5, 2019
Merged

[For 10.4] Fix "files:transfer-ownership" in S3 multibucket setups #36464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
87129f7
Revert "Skip testTransferAllFiles when on ObjectStoreStorage"
phil-davis Nov 22, 2019
a5215bd
Fix "files:transfer-ownership" in S3 multibucket setups
jvillafanez Oct 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 12 additions & 15 deletions apps/files/tests/Command/TransferOwnershipTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,9 @@
namespace OCA\Files\Tests\Command;

use OC\Encryption\Manager;
use OC\Files\ObjectStore\ObjectStoreStorage;
use OC\Files\View;
use OC\Share20\ProviderFactory;
use OCA\Files\Command\TransferOwnership;
use OCP\Files\Mount\IMountManager;
use OCP\Files\Storage;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Share;
Expand Down Expand Up @@ -166,10 +163,18 @@ private function createTestFilesForSourceUser() {
->setShareType(Share::SHARE_TYPE_USER)
->setPermissions(19);
$this->shareManager->createShare($share);

$subFolder = $userFolder->get('transfer/sub_folder');
$share = $this->shareManager->newShare();
$share->setNode($subFolder)
->setSharedBy('source-user')
->setSharedWith('share-receiver')
->setShareType(Share::SHARE_TYPE_USER)
->setPermissions(19);
$this->shareManager->createShare($share);
}

public function testTransferAllFiles() {
$this->markTestSkippedIfMultiBucketObjectStorage();
$this->encryptionManager->method('isReadyForUser')->willReturn(true);
$input = [
'source-user' => $this->sourceUser->getUID(),
Expand All @@ -182,13 +187,13 @@ public function testTransferAllFiles() {
$sourceShares = $this->shareManager->getSharesBy($this->sourceUser->getUID(), Share::SHARE_TYPE_USER);
$targetShares = $this->shareManager->getSharesBy($this->targetUser->getUID(), Share::SHARE_TYPE_USER);
$this->assertCount(0, $sourceShares);
$this->assertCount(3, $targetShares);
$this->assertCount(4, $targetShares);
}

public function folderPathProvider() {
return [
['transfer', 1, 2],
['transfer/sub_folder', 2, 1]
['transfer', 1, 3],
['transfer/sub_folder', 2, 2]
];
}

Expand All @@ -215,12 +220,4 @@ public function testTransferSpecificFolder($path, $expectedSourceShareCount, $ex
$this->assertCount($expectedSourceShareCount, $sourceShares);
$this->assertCount($expectedTargetShareCount, $targetShares);
}
private function markTestSkippedIfMultiBucketObjectStorage() {
$sourceUserView = new View('/source-user/files/');
/** @var Storage $storage */
list($storage, $internalPath) = $sourceUserView->resolvePath('transfer/test_file1');
if ($storage->instanceOfStorage(ObjectStoreStorage::class)) {
$this->markTestSkipped('transfer ownership may not work in a multi-bucket object store setup');
}
}
}
9 changes: 9 additions & 0 deletions changelog/unreleased/36464
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Bugfix: Fix "files:transfer-ownership" in S3 multibucket setups

There were problems using the files:transfer-ownership in setups using files_primary_s3
against S3 storage with multibucket configuration, when some of the transferred files
were shared with other people. This PR fixes that problem with the shares while
transferring the files, allowing the files:transfer-ownership to finish correctly

https://github.com/owncloud/core/pull/36464

135 changes: 105 additions & 30 deletions lib/private/Files/ObjectStore/ObjectStoreStorage.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
* @var array
*/
private static $tmpFiles = [];
/** @var array */
private $movingBetweenBuckets = [];
/**
* @var \OCP\Files\ObjectStore\IObjectStore $objectStore
*/
Expand Down Expand Up @@ -72,6 +74,27 @@ public function __construct($params) {
}
}

/**
* This is intended to be used during the moveFromStorage call. While moving, this is needed
* for the sourceStorage to know we're moving stuff and it shouldn't change the cache
* until it's finished.
* DO NOT USE OUTSIDE OF THIS CLASS
*/
public function setMovingBetweenBucketsInfo(array $info) {
$this->movingBetweenBuckets = $info;
}

/**
* This is intended to be used during the moveFromStorage call. While moving, this is needed
* for the sourceStorage to know we're moving stuff and it shouldn't change the cache
* until it's finished. This will be called when we finish moving all the files, in order
* for the sourceStorage to operate normally.
* DO NOT USE OUTSIDE OF THIS CLASS
*/
public function clearMovingBetweenBucketsInfo() {
$this->movingBetweenBuckets = [];
}

public function mkdir($path) {
$path = $this->normalizePath($path);

Expand All @@ -90,7 +113,9 @@ public function mkdir($path) {
if ($path === '') {
//create root on the fly
$data['etag'] = $this->getETag('');
$this->getCache()->put('', $data);
if (empty($this->movingBetweenBuckets)) {
$this->getCache()->put('', $data);
}
return true;
} else {
// if parent does not exist, create it
Expand All @@ -105,12 +130,14 @@ public function mkdir($path) {
// parent is a file
return false;
}
// finally create the new dir
$mTime = \time(); // update mtime
$data['mtime'] = $mTime;
$data['storage_mtime'] = $mTime;
$data['etag'] = $this->getETag($path);
$this->getCache()->put($path, $data);
if (empty($this->movingBetweenBuckets)) {
// finally create the new dir
$mTime = \time(); // update mtime
$data['mtime'] = $mTime;
$data['storage_mtime'] = $mTime;
$data['etag'] = $this->getETag($path);
$this->getCache()->put($path, $data);
}
return true;
}
}
Expand Down Expand Up @@ -168,7 +195,9 @@ public function rmdir($path) {

$this->rmObjects($path);

$this->getCache()->remove($path);
if (empty($this->movingBetweenBuckets)) {
$this->getCache()->remove($path);
}

return true;
}
Expand Down Expand Up @@ -202,7 +231,9 @@ public function unlink($path) {
//removing from cache is ok as it does not exist in the objectstore anyway
}
}
$this->getCache()->remove($path);
if (empty($this->movingBetweenBuckets)) {
$this->getCache()->remove($path);
}
return true;
}
return false;
Expand Down Expand Up @@ -305,6 +336,12 @@ public function fopen($path, $mode) {
\file_put_contents($tmpFile, $source);
}
self::$tmpFiles[$tmpFile] = $path;
if (isset($this->movingBetweenBuckets[$this->getBucket()])) {
// if we're moving files, mark the path we're moving. This is needed because
// we need to know the fileid of the file we're moving in order to create
// the new file with the same name in the other bucket
$this->movingBetweenBuckets[$this->getBucket()]['paths'][$path] = true;
}

return \fopen('close://' . $tmpFile, $mode);
}
Expand All @@ -320,7 +357,9 @@ public function rename($source, $target) {
$source = $this->normalizePath($source);
$target = $this->normalizePath($target);
$this->remove($target);
$this->getCache()->move($source, $target);
if (empty($this->movingBetweenBuckets)) {
$this->getCache()->move($source, $target);
}
$this->touch(\dirname($target));
return true;
}
Expand All @@ -340,7 +379,22 @@ public function moveFromStorage(\OCP\Files\Storage $sourceStorage, $sourceIntern
/** @var ObjectStoreStorage $sourceStorage */
'@phan-var ObjectStoreStorage $sourceStorage';
if ($this->getBucket() !== $sourceStorage->getBucket()) {
return parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
$this->movingBetweenBuckets[$this->getBucket()] = [
'sourceStorage' => $sourceStorage,
'sourceBase' => $sourceInternalPath,
'targetBase' => $targetInternalPath,
'paths' => [],
];
$sourceStorage->setMovingBetweenBucketsInfo($this->movingBetweenBuckets);
try {
$result = parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
$this->getCache()->moveFromCache($sourceStorage->getCache(), $sourceInternalPath, $targetInternalPath);
} finally {
// ensure we restore the normal behaviour in both storages
$sourceStorage->clearMovingBetweenBucketsInfo();
unset($this->movingBetweenBuckets[$this->getBucket()]);
}
return $result;
}

// source and target live on the same object store and we can simply rename
Expand Down Expand Up @@ -377,24 +431,27 @@ public function touch($path, $mtime = null) {
$stat['mtime'] = $mtime;
$this->getCache()->update($stat['fileid'], $stat);
} else {
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
// create new file
$stat = [
'etag' => $this->getETag($path),
'mimetype' => $mimeType,
'size' => 0,
'mtime' => $mtime,
'storage_mtime' => $mtime,
'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
];
$fileId = $this->getCache()->put($path, $stat);
try {
//read an empty file from memory
$this->objectStore->writeObject($this->getURN($fileId), \fopen('php://memory', 'r'));
} catch (\Exception $ex) {
$this->getCache()->remove($path);
\OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR);
return false;
if (!isset($this->movingBetweenBuckets[$this->getBucket()]['paths'][$path])) {
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
// create new file
$stat = [
'etag' => $this->getETag($path),
'mimetype' => $mimeType,
'size' => 0,
'mtime' => $mtime,
'storage_mtime' => $mtime,
'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
];

$fileId = $this->getCache()->put($path, $stat);
try {
//read an empty file from memory
$this->objectStore->writeObject($this->getURN($fileId), \fopen('php://memory', 'r'));
} catch (\Exception $ex) {
$this->getCache()->remove($path);
\OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR);
return false;
}
}
}
return true;
Expand All @@ -421,7 +478,25 @@ public function writeBack($tmpFile) {
$stat['mimetype'] = \OC::$server->getMimeTypeDetector()->detect($tmpFile);
$stat['etag'] = $this->getETag($path);

$fileId = $this->getCache()->put($path, $stat);
if (isset($this->movingBetweenBuckets[$this->getBucket()]['paths'][$path])) {
// if we're moving, we need the fileid of the old entry in order to create the new
// file with the same name. The "movingBetweenBuckets" should contain enough
// information to get the fileid of the old entry from the filecache.
$targetBase = $this->movingBetweenBuckets[$this->getBucket()]['targetBase'];
$sourceBase = $this->movingBetweenBuckets[$this->getBucket()]['sourceBase'];
if (\substr($path, 0, \strlen($targetBase)) === $targetBase) {
$movingPath = \substr($path, \strlen($targetBase));
$originalSource = $sourceBase . $movingPath;
$originalStorage = $this->movingBetweenBuckets[$this->getBucket()]['sourceStorage'];
$fileId = $originalStorage->stat($originalSource)['fileid'];
} else {
// if the target path is outside the base... this shouldn't happen, but fallback to normal behaviour
$fileId = $this->getCache()->put($path, $stat);
}
unset($this->movingBetweenBuckets[$this->getBucket()]['paths'][$path]);
} else {
$fileId = $this->getCache()->put($path, $stat);
}
try {
//upload to object storage
$this->objectStore->writeObject($this->getURN($fileId), \fopen($tmpFile, 'r'));
Expand Down
X Tutup