migrate therinaldos.com data
Build & Deploy to DigitalOcean Space / build (push) Failing after 2m38s

This commit is contained in:
2024-05-05 15:50:45 -04:00
commit ef1ff240d4
23182 changed files with 3801898 additions and 0 deletions
@@ -0,0 +1,113 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
* @author Christoph Wurst <christoph@winzerhof-wurst.at>
* @author Robin Appelman <robin@icewind.nl>
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Lock;
use OCP\Lock\ILockingProvider;
/**
* Base locking provider that keeps track of locks acquired during the current request
* to release any leftover locks at the end of the request
*/
abstract class AbstractLockingProvider implements ILockingProvider {
protected array $acquiredLocks = [
'shared' => [],
'exclusive' => []
];
/**
*
* @param int $ttl how long until we clear stray locks in seconds
*/
public function __construct(protected int $ttl) {
}
/** @inheritDoc */
protected function hasAcquiredLock(string $path, int $type): bool {
if ($type === self::LOCK_SHARED) {
return isset($this->acquiredLocks['shared'][$path]) && $this->acquiredLocks['shared'][$path] > 0;
} else {
return isset($this->acquiredLocks['exclusive'][$path]) && $this->acquiredLocks['exclusive'][$path] === true;
}
}
/** @inheritDoc */
protected function markAcquire(string $path, int $targetType): void {
if ($targetType === self::LOCK_SHARED) {
if (!isset($this->acquiredLocks['shared'][$path])) {
$this->acquiredLocks['shared'][$path] = 0;
}
$this->acquiredLocks['shared'][$path]++;
} else {
$this->acquiredLocks['exclusive'][$path] = true;
}
}
/** @inheritDoc */
protected function markRelease(string $path, int $type): void {
if ($type === self::LOCK_SHARED) {
if (isset($this->acquiredLocks['shared'][$path]) and $this->acquiredLocks['shared'][$path] > 0) {
$this->acquiredLocks['shared'][$path]--;
if ($this->acquiredLocks['shared'][$path] === 0) {
unset($this->acquiredLocks['shared'][$path]);
}
}
} elseif ($type === self::LOCK_EXCLUSIVE) {
unset($this->acquiredLocks['exclusive'][$path]);
}
}
/** @inheritDoc */
protected function markChange(string $path, int $targetType): void {
if ($targetType === self::LOCK_SHARED) {
unset($this->acquiredLocks['exclusive'][$path]);
if (!isset($this->acquiredLocks['shared'][$path])) {
$this->acquiredLocks['shared'][$path] = 0;
}
$this->acquiredLocks['shared'][$path]++;
} elseif ($targetType === self::LOCK_EXCLUSIVE) {
$this->acquiredLocks['exclusive'][$path] = true;
$this->acquiredLocks['shared'][$path]--;
}
}
/** @inheritDoc */
public function releaseAll(): void {
foreach ($this->acquiredLocks['shared'] as $path => $count) {
for ($i = 0; $i < $count; $i++) {
$this->releaseLock($path, self::LOCK_SHARED);
}
}
foreach ($this->acquiredLocks['exclusive'] as $path => $hasLock) {
$this->releaseLock($path, self::LOCK_EXCLUSIVE);
}
}
protected function getOwnSharedLockCount(string $path): int {
return $this->acquiredLocks['shared'][$path] ?? 0;
}
}
@@ -0,0 +1,261 @@
<?php
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
* @author Christoph Wurst <christoph@winzerhof-wurst.at>
* @author Individual IT Services <info@individual-it.net>
* @author Joas Schilling <coding@schilljs.com>
* @author Morris Jobke <hey@morrisjobke.de>
* @author Ole Ostergaard <ole.c.ostergaard@gmail.com>
* @author Robin Appelman <robin@icewind.nl>
* @author Roeland Jago Douma <roeland@famdouma.nl>
* @author Carl Schwan <carl@carlschwan.eu>
*
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Lock;
use OC\DB\QueryBuilder\Literal;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use OCP\Lock\ILockingProvider;
use OCP\Lock\LockedException;
/**
* Locking provider that stores the locks in the database
*/
class DBLockingProvider extends AbstractLockingProvider {
private array $sharedLocks = [];
public function __construct(
private IDBConnection $connection,
private ITimeFactory $timeFactory,
int $ttl = 3600,
private bool $cacheSharedLocks = true
) {
parent::__construct($ttl);
}
/**
* Check if we have an open shared lock for a path
*/
protected function isLocallyLocked(string $path): bool {
return isset($this->sharedLocks[$path]) && $this->sharedLocks[$path];
}
/** @inheritDoc */
protected function markAcquire(string $path, int $targetType): void {
parent::markAcquire($path, $targetType);
if ($this->cacheSharedLocks) {
if ($targetType === self::LOCK_SHARED) {
$this->sharedLocks[$path] = true;
}
}
}
/**
* Change the type of an existing tracked lock
*/
protected function markChange(string $path, int $targetType): void {
parent::markChange($path, $targetType);
if ($this->cacheSharedLocks) {
if ($targetType === self::LOCK_SHARED) {
$this->sharedLocks[$path] = true;
} elseif ($targetType === self::LOCK_EXCLUSIVE) {
$this->sharedLocks[$path] = false;
}
}
}
/**
* Insert a file locking row if it does not exists.
*/
protected function initLockField(string $path, int $lock = 0): int {
$expire = $this->getExpireTime();
return $this->connection->insertIgnoreConflict('file_locks', [
'key' => $path,
'lock' => $lock,
'ttl' => $expire
]);
}
protected function getExpireTime(): int {
return $this->timeFactory->getTime() + $this->ttl;
}
/** @inheritDoc */
public function isLocked(string $path, int $type): bool {
if ($this->hasAcquiredLock($path, $type)) {
return true;
}
$query = $this->connection->getQueryBuilder();
$query->select('lock')
->from('file_locks')
->where($query->expr()->eq('key', $query->createNamedParameter($path)));
$result = $query->executeQuery();
$lockValue = (int)$result->fetchOne();
if ($type === self::LOCK_SHARED) {
if ($this->isLocallyLocked($path)) {
// if we have a shared lock we kept open locally but it's released we always have at least 1 shared lock in the db
return $lockValue > 1;
} else {
return $lockValue > 0;
}
} elseif ($type === self::LOCK_EXCLUSIVE) {
return $lockValue === -1;
} else {
return false;
}
}
/** @inheritDoc */
public function acquireLock(string $path, int $type, ?string $readablePath = null): void {
$expire = $this->getExpireTime();
if ($type === self::LOCK_SHARED) {
if (!$this->isLocallyLocked($path)) {
$result = $this->initLockField($path, 1);
if ($result <= 0) {
$query = $this->connection->getQueryBuilder();
$query->update('file_locks')
->set('lock', $query->func()->add('lock', $query->createNamedParameter(1, IQueryBuilder::PARAM_INT)))
->set('ttl', $query->createNamedParameter($expire))
->where($query->expr()->eq('key', $query->createNamedParameter($path)))
->andWhere($query->expr()->gte('lock', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
$result = $query->executeStatement();
}
} else {
$result = 1;
}
} else {
$existing = 0;
if ($this->hasAcquiredLock($path, ILockingProvider::LOCK_SHARED) === false && $this->isLocallyLocked($path)) {
$existing = 1;
}
$result = $this->initLockField($path, -1);
if ($result <= 0) {
$query = $this->connection->getQueryBuilder();
$query->update('file_locks')
->set('lock', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))
->set('ttl', $query->createNamedParameter($expire, IQueryBuilder::PARAM_INT))
->where($query->expr()->eq('key', $query->createNamedParameter($path)))
->andWhere($query->expr()->eq('lock', $query->createNamedParameter($existing)));
$result = $query->executeStatement();
}
}
if ($result !== 1) {
throw new LockedException($path, null, null, $readablePath);
}
$this->markAcquire($path, $type);
}
/** @inheritDoc */
public function releaseLock(string $path, int $type): void {
$this->markRelease($path, $type);
// we keep shared locks till the end of the request so we can re-use them
if ($type === self::LOCK_EXCLUSIVE) {
$qb = $this->connection->getQueryBuilder();
$qb->update('file_locks')
->set('lock', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))
->where($qb->expr()->eq('key', $qb->createNamedParameter($path)))
->andWhere($qb->expr()->eq('lock', $qb->createNamedParameter(-1, IQueryBuilder::PARAM_INT)));
$qb->executeStatement();
} elseif (!$this->cacheSharedLocks) {
$qb = $this->connection->getQueryBuilder();
$qb->update('file_locks')
->set('lock', $qb->func()->subtract('lock', $qb->createNamedParameter(1, IQueryBuilder::PARAM_INT)))
->where($qb->expr()->eq('key', $qb->createNamedParameter($path)))
->andWhere($qb->expr()->gt('lock', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
$qb->executeStatement();
}
}
/** @inheritDoc */
public function changeLock(string $path, int $targetType): void {
$expire = $this->getExpireTime();
if ($targetType === self::LOCK_SHARED) {
$qb = $this->connection->getQueryBuilder();
$result = $qb->update('file_locks')
->set('lock', $qb->createNamedParameter(1, IQueryBuilder::PARAM_INT))
->set('ttl', $qb->createNamedParameter($expire, IQueryBuilder::PARAM_INT))
->where($qb->expr()->andX(
$qb->expr()->eq('key', $qb->createNamedParameter($path)),
$qb->expr()->eq('lock', $qb->createNamedParameter(-1, IQueryBuilder::PARAM_INT))
))->executeStatement();
} else {
// since we only keep one shared lock in the db we need to check if we have more then one shared lock locally manually
if (isset($this->acquiredLocks['shared'][$path]) && $this->acquiredLocks['shared'][$path] > 1) {
throw new LockedException($path);
}
$qb = $this->connection->getQueryBuilder();
$result = $qb->update('file_locks')
->set('lock', $qb->createNamedParameter(-1, IQueryBuilder::PARAM_INT))
->set('ttl', $qb->createNamedParameter($expire, IQueryBuilder::PARAM_INT))
->where($qb->expr()->andX(
$qb->expr()->eq('key', $qb->createNamedParameter($path)),
$qb->expr()->eq('lock', $qb->createNamedParameter(1, IQueryBuilder::PARAM_INT))
))->executeStatement();
}
if ($result !== 1) {
throw new LockedException($path);
}
$this->markChange($path, $targetType);
}
/** @inheritDoc */
public function cleanExpiredLocks(): void {
$expire = $this->timeFactory->getTime();
try {
$qb = $this->connection->getQueryBuilder();
$qb->delete('file_locks')
->where($qb->expr()->lt('ttl', $qb->createNamedParameter($expire, IQueryBuilder::PARAM_INT)))
->executeStatement();
} catch (\Exception $e) {
// If the table is missing, the clean up was successful
if ($this->connection->tableExists('file_locks')) {
throw $e;
}
}
}
/** @inheritDoc */
public function releaseAll(): void {
parent::releaseAll();
if (!$this->cacheSharedLocks) {
return;
}
// since we keep shared locks we need to manually clean those
$lockedPaths = array_keys($this->sharedLocks);
$lockedPaths = array_filter($lockedPaths, function ($path) {
return $this->sharedLocks[$path];
});
$chunkedPaths = array_chunk($lockedPaths, 100);
$qb = $this->connection->getQueryBuilder();
$qb->update('file_locks')
->set('lock', $qb->func()->subtract('lock', $qb->expr()->literal(1)))
->where($qb->expr()->in('key', $qb->createParameter('chunk')))
->andWhere($qb->expr()->gt('lock', new Literal(0)));
foreach ($chunkedPaths as $chunk) {
$qb->setParameter('chunk', $chunk, IQueryBuilder::PARAM_STR_ARRAY);
$qb->executeStatement();
}
}
}
@@ -0,0 +1,200 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
* @author Christoph Wurst <christoph@winzerhof-wurst.at>
* @author Jaakko Salo <jaakkos@gmail.com>
* @author Robin Appelman <robin@icewind.nl>
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Lock;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\IMemcache;
use OCP\IMemcacheTTL;
use OCP\Lock\LockedException;
class MemcacheLockingProvider extends AbstractLockingProvider {
/** @var array<string, array{time: int, ttl: int}> */
private array $oldTTLs = [];
public function __construct(
private IMemcache $memcache,
private ITimeFactory $timeFactory,
int $ttl = 3600,
) {
parent::__construct($ttl);
}
private function setTTL(string $path, int $ttl = null, ?int $compare = null): void {
if (is_null($ttl)) {
$ttl = $this->ttl;
}
if ($this->memcache instanceof IMemcacheTTL) {
if ($compare !== null) {
$this->memcache->compareSetTTL($path, $compare, $ttl);
} else {
$this->memcache->setTTL($path, $ttl);
}
}
}
private function getTTL(string $path): int {
if ($this->memcache instanceof IMemcacheTTL) {
$ttl = $this->memcache->getTTL($path);
return $ttl === false ? -1 : $ttl;
} else {
return -1;
}
}
public function isLocked(string $path, int $type): bool {
$lockValue = $this->memcache->get($path);
if ($type === self::LOCK_SHARED) {
return is_int($lockValue) && $lockValue > 0;
} elseif ($type === self::LOCK_EXCLUSIVE) {
return $lockValue === 'exclusive';
} else {
return false;
}
}
public function acquireLock(string $path, int $type, ?string $readablePath = null): void {
if ($type === self::LOCK_SHARED) {
// save the old TTL to for `restoreTTL`
$this->oldTTLs[$path] = [
"ttl" => $this->getTTL($path),
"time" => $this->timeFactory->getTime()
];
if (!$this->memcache->inc($path)) {
throw new LockedException($path, null, $this->getExistingLockForException($path), $readablePath);
}
} else {
// when getting exclusive locks, we know there are no old TTLs to restore
$this->memcache->add($path, 0);
// ttl is updated automatically when the `set` succeeds
if (!$this->memcache->cas($path, 0, 'exclusive')) {
throw new LockedException($path, null, $this->getExistingLockForException($path), $readablePath);
}
unset($this->oldTTLs[$path]);
}
$this->setTTL($path);
$this->markAcquire($path, $type);
}
public function releaseLock(string $path, int $type): void {
if ($type === self::LOCK_SHARED) {
$ownSharedLockCount = $this->getOwnSharedLockCount($path);
$newValue = 0;
if ($ownSharedLockCount === 0) { // if we are not holding the lock, don't try to release it
return;
}
if ($ownSharedLockCount === 1) {
$removed = $this->memcache->cad($path, 1); // if we're the only one having a shared lock we can remove it in one go
if (!$removed) { //someone else also has a shared lock, decrease only
$newValue = $this->memcache->dec($path);
}
} else {
// if we own more than one lock ourselves just decrease
$newValue = $this->memcache->dec($path);
}
if ($newValue > 0) {
$this->restoreTTL($path);
} else {
unset($this->oldTTLs[$path]);
}
// if we somehow release more locks then exists, reset the lock
if ($newValue < 0) {
$this->memcache->cad($path, $newValue);
}
} elseif ($type === self::LOCK_EXCLUSIVE) {
$this->memcache->cad($path, 'exclusive');
}
$this->markRelease($path, $type);
}
public function changeLock(string $path, int $targetType): void {
if ($targetType === self::LOCK_SHARED) {
if (!$this->memcache->cas($path, 'exclusive', 1)) {
throw new LockedException($path, null, $this->getExistingLockForException($path));
}
} elseif ($targetType === self::LOCK_EXCLUSIVE) {
// we can only change a shared lock to an exclusive if there's only a single owner of the shared lock
if (!$this->memcache->cas($path, 1, 'exclusive')) {
$this->restoreTTL($path);
throw new LockedException($path, null, $this->getExistingLockForException($path));
}
unset($this->oldTTLs[$path]);
}
$this->setTTL($path);
$this->markChange($path, $targetType);
}
/**
* With shared locks, each time the lock is acquired, the ttl for the path is reset.
*
* Due to this "ttl extension" when a shared lock isn't freed correctly for any reason
* the lock won't expire until no shared locks are required for the path for 1h.
* This can lead to a client repeatedly trying to upload a file, and failing forever
* because the lock never gets the opportunity to expire.
*
* To help the lock expire in this case, we lower the TTL back to what it was before we
* took the shared lock *only* if nobody else got a shared lock after we did.
*
* This doesn't handle all cases where multiple requests are acquiring shared locks
* but it should handle some of the more common ones and not hurt things further
*/
private function restoreTTL(string $path): void {
if (isset($this->oldTTLs[$path])) {
$saved = $this->oldTTLs[$path];
$elapsed = $this->timeFactory->getTime() - $saved['time'];
// old value to compare to when setting ttl in case someone else changes the lock in the middle of this function
$value = $this->memcache->get($path);
$currentTtl = $this->getTTL($path);
// what the old ttl would be given the time elapsed since we acquired the lock
// note that if this gets negative the key will be expired directly when we set the ttl
$remainingOldTtl = $saved['ttl'] - $elapsed;
// what the currently ttl would be if nobody else acquired a lock since we did (+1 to cover rounding errors)
$expectedTtl = $this->ttl - $elapsed + 1;
// check if another request has acquired a lock (and didn't release it yet)
if ($currentTtl <= $expectedTtl) {
$this->setTTL($path, $remainingOldTtl, $value);
}
}
}
private function getExistingLockForException(string $path): string {
$existing = $this->memcache->get($path);
if (!$existing) {
return 'none';
} elseif ($existing === 'exclusive') {
return $existing;
} else {
return $existing . ' shared locks';
}
}
}
@@ -0,0 +1,71 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
* @author Robin Appelman <robin@icewind.nl>
* @author Roeland Jago Douma <roeland@famdouma.nl>
* @author Vincent Petry <vincent@nextcloud.com>
*
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Lock;
use OCP\Lock\ILockingProvider;
/**
* Locking provider that does nothing.
*
* To be used when locking is disabled.
*/
class NoopLockingProvider implements ILockingProvider {
/**
* {@inheritdoc}
*/
public function isLocked(string $path, int $type): bool {
return false;
}
/**
* {@inheritdoc}
*/
public function acquireLock(string $path, int $type, ?string $readablePath = null): void {
// do nothing
}
/**
* {@inheritdoc}
*/
public function releaseLock(string $path, int $type): void {
// do nothing
}
/**1
* {@inheritdoc}
*/
public function releaseAll(): void {
// do nothing
}
/**
* {@inheritdoc}
*/
public function changeLock(string $path, int $targetType): void {
// do nothing
}
}