migrate therinaldos.com data

wedding-pics/core/Command/Preview/Generate.php

@@ -0,0 +1,137 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Command\Preview;
+
+use OCP\Files\Config\IUserMountCache;
+use OCP\Files\File;
+use OCP\Files\IRootFolder;
+use OCP\Files\Node;
+use OCP\Files\NotFoundException;
+use OCP\IPreview;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Generate extends Command {
+	public function __construct(
+		private IRootFolder $rootFolder,
+		private IUserMountCache $userMountCache,
+		private IPreview $previewManager,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this
+			->setName('preview:generate')
+			->setDescription('generate a preview for a file')
+			->addArgument("file", InputArgument::REQUIRED, "path or fileid of the file to generate the preview for")
+			->addOption("size", "s", InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED, "size to generate the preview for in pixels, defaults to 64x64", ["64x64"])
+			->addOption("crop", "c", InputOption::VALUE_NONE, "crop the previews instead of maintaining aspect ratio")
+			->addOption("mode", "m", InputOption::VALUE_REQUIRED, "mode for generating uncropped previews, 'cover' or 'fill'", IPreview::MODE_FILL);
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$fileInput = $input->getArgument("file");
+		$sizes = $input->getOption("size");
+		$sizes = array_map(function (string $size) use ($output, &$error) {
+			if (str_contains($size, 'x')) {
+				$sizeParts = explode('x', $size, 2);
+			} else {
+				$sizeParts = [$size, $size];
+			}
+			if (!is_numeric($sizeParts[0]) || !is_numeric($sizeParts[1])) {
+				$output->writeln("<error>Invalid size $size</error>");
+				return null;
+			}
+
+			return array_map("intval", $sizeParts);
+		}, $sizes);
+		if (in_array(null, $sizes)) {
+			return 1;
+		}
+
+		$mode = $input->getOption("mode");
+		if ($mode !== IPreview::MODE_FILL && $mode !== IPreview::MODE_COVER) {
+			$output->writeln("<error>Invalid mode $mode</error>");
+			return 1;
+		}
+		$crop = $input->getOption("crop");
+		$file = $this->getFile($fileInput);
+		if (!$file) {
+			$output->writeln("<error>File $fileInput not found</error>");
+			return 1;
+		}
+		if (!$file instanceof File) {
+			$output->writeln("<error>Can't generate previews for folders</error>");
+			return 1;
+		}
+
+		if (!$this->previewManager->isAvailable($file)) {
+			$output->writeln("<error>No preview generator available for file of type" . $file->getMimetype() . "</error>");
+			return 1;
+		}
+
+		$specifications = array_map(function (array $sizes) use ($crop, $mode) {
+			return [
+				'width' => $sizes[0],
+				'height' => $sizes[1],
+				'crop' => $crop,
+				'mode' => $mode,
+			];
+		}, $sizes);
+
+		$this->previewManager->generatePreviews($file, $specifications);
+		if (count($specifications) > 1) {
+			$output->writeln("generated <info>" . count($specifications) . "</info> previews");
+		} else {
+			$output->writeln("preview generated");
+		}
+		return 0;
+	}
+
+	private function getFile(string $fileInput): ?Node {
+		if (is_numeric($fileInput)) {
+			$mounts = $this->userMountCache->getMountsForFileId((int)$fileInput);
+			if (!$mounts) {
+				return null;
+			}
+			$mount = $mounts[0];
+			$userFolder = $this->rootFolder->getUserFolder($mount->getUser()->getUID());
+			$nodes = $userFolder->getById((int)$fileInput);
+			if (!$nodes) {
+				return null;
+			}
+			return $nodes[0];
+		} else {
+			try {
+				return $this->rootFolder->get($fileInput);
+			} catch (NotFoundException $e) {
+				return null;
+			}
+		}
+	}
+}

wedding-pics/core/Command/Preview/Repair.php

@@ -0,0 +1,309 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
+ *
+ * @author Christoph Wurst <christoph@winzerhof-wurst.at>
+ * @author Morris Jobke <hey@morrisjobke.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OC\Core\Command\Preview;
+
+use bantu\IniGetWrapper\IniGetWrapper;
+use OC\Preview\Storage\Root;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use OCP\IConfig;
+use OCP\Lock\ILockingProvider;
+use OCP\Lock\LockedException;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
+
+use function pcntl_signal;
+
+class Repair extends Command {
+	private bool $stopSignalReceived = false;
+	private int $memoryLimit;
+	private int $memoryTreshold;
+
+	public function __construct(
+		protected IConfig $config,
+		private IRootFolder $rootFolder,
+		private LoggerInterface $logger,
+		IniGetWrapper $phpIni,
+		private ILockingProvider $lockingProvider,
+	) {
+		$this->memoryLimit = (int)$phpIni->getBytes('memory_limit');
+		$this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024;
+
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this
+			->setName('preview:repair')
+			->setDescription('distributes the existing previews into subfolders')
+			->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.')
+			->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.')
+			->addOption('delete', null, InputOption::VALUE_NONE, 'Delete instead of migrating them. Usefull if too many entries to migrate.');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		if ($this->memoryLimit !== -1) {
+			$limitInMiB = round($this->memoryLimit / 1024 / 1024, 1);
+			$thresholdInMiB = round($this->memoryTreshold / 1024 / 1024, 1);
+			$output->writeln("Memory limit is $limitInMiB MiB");
+			$output->writeln("Memory threshold is $thresholdInMiB MiB");
+			$output->writeln("");
+			$memoryCheckEnabled = true;
+		} else {
+			$output->writeln("No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.");
+			$output->writeln("");
+			$memoryCheckEnabled = false;
+		}
+
+		$dryMode = $input->getOption('dry');
+		$deleteMode = $input->getOption('delete');
+
+
+		if ($dryMode) {
+			$output->writeln("INFO: The migration is run in dry mode and will not modify anything.");
+			$output->writeln("");
+		} elseif ($deleteMode) {
+			$output->writeln("WARN: The migration will _DELETE_ old previews.");
+			$output->writeln("");
+		}
+
+		$instanceId = $this->config->getSystemValueString('instanceid');
+
+		$output->writeln("This will migrate all previews from the old preview location to the new one.");
+		$output->writeln('');
+
+		$output->writeln('Fetching previews that need to be migrated …');
+		/** @var \OCP\Files\Folder $currentPreviewFolder */
+		$currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview");
+
+		$directoryListing = $currentPreviewFolder->getDirectoryListing();
+
+		$total = count($directoryListing);
+		/**
+		 * by default there could be 0-9 a-f and the old-multibucket folder which are all fine
+		 */
+		if ($total < 18) {
+			$directoryListing = array_filter($directoryListing, function ($dir) {
+				if ($dir->getName() === 'old-multibucket') {
+					return false;
+				}
+
+				// a-f can't be a file ID -> removing from migration
+				if (preg_match('!^[a-f]$!', $dir->getName())) {
+					return false;
+				}
+
+				if (preg_match('!^[0-9]$!', $dir->getName())) {
+					// ignore folders that only has folders in them
+					if ($dir instanceof Folder) {
+						foreach ($dir->getDirectoryListing() as $entry) {
+							if (!$entry instanceof Folder) {
+								return true;
+							}
+						}
+						return false;
+					}
+				}
+				return true;
+			});
+			$total = count($directoryListing);
+		}
+
+		if ($total === 0) {
+			$output->writeln("All previews are already migrated.");
+			return 0;
+		}
+
+		$output->writeln("A total of $total preview files need to be migrated.");
+		$output->writeln("");
+		$output->writeln("The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This will finish the current batch and then stop the migration. This migration can then just be started and it will continue.");
+
+		if ($input->getOption('batch')) {
+			$output->writeln('Batch mode active: migration is started right away.');
+		} else {
+			$helper = $this->getHelper('question');
+			$question = new ConfirmationQuestion('<info>Should the migration be started? (y/[n]) </info>', false);
+
+			if (!$helper->ask($input, $output, $question)) {
+				return 0;
+			}
+		}
+
+		// register the SIGINT listener late in here to be able to exit in the early process of this command
+		pcntl_signal(SIGINT, [$this, 'sigIntHandler']);
+
+		$output->writeln("");
+		$output->writeln("");
+		$section1 = $output->section();
+		$section2 = $output->section();
+		$progressBar = new ProgressBar($section2, $total);
+		$progressBar->setFormat("%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%");
+		$time = (new \DateTime())->format('H:i:s');
+		$progressBar->setMessage("$time Starting …");
+		$progressBar->maxSecondsBetweenRedraws(0.2);
+		$progressBar->start();
+
+		foreach ($directoryListing as $oldPreviewFolder) {
+			pcntl_signal_dispatch();
+			$name = $oldPreviewFolder->getName();
+			$time = (new \DateTime())->format('H:i:s');
+			$section1->writeln("$time Migrating previews of file with fileId $name …");
+			$progressBar->display();
+
+			if ($this->stopSignalReceived) {
+				$section1->writeln("$time Stopping migration …");
+				return 0;
+			}
+			if (!$oldPreviewFolder instanceof Folder) {
+				$section1->writeln("         Skipping non-folder $name …");
+				$progressBar->advance();
+				continue;
+			}
+			if ($name === 'old-multibucket') {
+				$section1->writeln("         Skipping fallback mount point $name …");
+				$progressBar->advance();
+				continue;
+			}
+			if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) {
+				$section1->writeln("         Skipping hex-digit folder $name …");
+				$progressBar->advance();
+				continue;
+			}
+			if (!preg_match('!^\d+$!', $name)) {
+				$section1->writeln("         Skipping non-numeric folder $name …");
+				$progressBar->advance();
+				continue;
+			}
+
+			$newFoldername = Root::getInternalFolder($name);
+
+			$memoryUsage = memory_get_usage();
+			if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) {
+				$section1->writeln("");
+				$section1->writeln("");
+				$section1->writeln("");
+				$section1->writeln("         Stopped process 25 MB before reaching the memory limit to avoid a hard crash.");
+				$time = (new \DateTime())->format('H:i:s');
+				$section1->writeln("$time Reached memory limit and stopped to avoid hard crash.");
+				return 1;
+			}
+
+			$lockName = 'occ preview:repair lock ' . $oldPreviewFolder->getId();
+			try {
+				$section1->writeln("         Locking \"$lockName\" …", OutputInterface::VERBOSITY_VERBOSE);
+				$this->lockingProvider->acquireLock($lockName, ILockingProvider::LOCK_EXCLUSIVE);
+			} catch (LockedException $e) {
+				$section1->writeln("         Skipping because it is locked - another process seems to work on this …");
+				continue;
+			}
+
+			$previews = $oldPreviewFolder->getDirectoryListing();
+			if ($previews !== []) {
+				try {
+					$this->rootFolder->get("appdata_$instanceId/preview/$newFoldername");
+				} catch (NotFoundException $e) {
+					$section1->writeln("         Create folder preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE);
+					if (!$dryMode) {
+						$this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername");
+					}
+				}
+
+				foreach ($previews as $preview) {
+					pcntl_signal_dispatch();
+					$previewName = $preview->getName();
+
+					if ($preview instanceof Folder) {
+						$section1->writeln("         Skipping folder $name/$previewName …");
+						$progressBar->advance();
+						continue;
+					}
+
+					// Execute process
+					if (!$dryMode) {
+						// Delete preview instead of moving
+						if ($deleteMode) {
+							try {
+								$section1->writeln("         Delete preview/$name/$previewName", OutputInterface::VERBOSITY_VERBOSE);
+								$preview->delete();
+							} catch (\Exception $e) {
+								$this->logger->error("Failed to delete preview at preview/$name/$previewName", [
+									'app' => 'core',
+									'exception' => $e,
+								]);
+							}
+						} else {
+							try {
+								$section1->writeln("         Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE);
+								$preview->move("appdata_$instanceId/preview/$newFoldername/$previewName");
+							} catch (\Exception $e) {
+								$this->logger->error("Failed to move preview from preview/$name/$previewName to preview/$newFoldername", [
+									'app' => 'core',
+									'exception' => $e,
+								]);
+							}
+						}
+					}
+				}
+			}
+
+			if ($oldPreviewFolder->getDirectoryListing() === []) {
+				$section1->writeln("         Delete empty folder preview/$name", OutputInterface::VERBOSITY_VERBOSE);
+				if (!$dryMode) {
+					try {
+						$oldPreviewFolder->delete();
+					} catch (\Exception $e) {
+						$this->logger->error("Failed to delete empty folder preview/$name", [
+							'app' => 'core',
+							'exception' => $e,
+						]);
+					}
+				}
+			}
+
+			$this->lockingProvider->releaseLock($lockName, ILockingProvider::LOCK_EXCLUSIVE);
+			$section1->writeln("         Unlocked", OutputInterface::VERBOSITY_VERBOSE);
+
+			$section1->writeln("         Finished migrating previews of file with fileId $name …");
+			$progressBar->advance();
+		}
+
+		$progressBar->finish();
+		$output->writeln("");
+		return 0;
+	}
+
+	protected function sigIntHandler() {
+		echo "\nSignal received - will finish the step and then stop the migration.\n\n\n";
+		$this->stopSignalReceived = true;
+	}
+}

wedding-pics/core/Command/Preview/ResetRenderedTexts.php

@@ -0,0 +1,181 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2021, Daniel Calviño Sánchez <danxuliu@gmail.com>
+ *
+ * @author Daniel Calviño Sánchez <danxuliu@gmail.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OC\Core\Command\Preview;
+
+use OC\Preview\Storage\Root;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\IMimeTypeLoader;
+use OCP\Files\NotFoundException;
+use OCP\Files\NotPermittedException;
+use OCP\IAvatarManager;
+use OCP\IDBConnection;
+use OCP\IUserManager;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class ResetRenderedTexts extends Command {
+	public function __construct(
+		protected IDBConnection $connection,
+		protected IUserManager $userManager,
+		protected IAvatarManager $avatarManager,
+		private Root $previewFolder,
+		private IMimeTypeLoader $mimeTypeLoader,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this
+			->setName('preview:reset-rendered-texts')
+			->setDescription('Deletes all generated avatars and previews of text and md files')
+			->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not delete any files - in combination with the verbose mode one could check the operations.');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$dryMode = $input->getOption('dry');
+
+		if ($dryMode) {
+			$output->writeln('INFO: The command is run in dry mode and will not modify anything.');
+			$output->writeln('');
+		}
+
+		$this->deleteAvatars($output, $dryMode);
+		$this->deletePreviews($output, $dryMode);
+
+		return 0;
+	}
+
+	private function deleteAvatars(OutputInterface $output, bool $dryMode): void {
+		$avatarsToDeleteCount = 0;
+
+		foreach ($this->getAvatarsToDelete() as [$userId, $avatar]) {
+			$output->writeln('Deleting avatar for ' . $userId, OutputInterface::VERBOSITY_VERBOSE);
+
+			$avatarsToDeleteCount++;
+
+			if ($dryMode) {
+				continue;
+			}
+
+			try {
+				$avatar->remove();
+			} catch (NotFoundException $e) {
+				// continue
+			} catch (NotPermittedException $e) {
+				// continue
+			}
+		}
+
+		$output->writeln('Deleted ' . $avatarsToDeleteCount . ' avatars');
+		$output->writeln('');
+	}
+
+	private function getAvatarsToDelete(): \Iterator {
+		foreach ($this->userManager->search('') as $user) {
+			$avatar = $this->avatarManager->getAvatar($user->getUID());
+
+			if (!$avatar->isCustomAvatar()) {
+				yield [$user->getUID(), $avatar];
+			}
+		}
+	}
+
+	private function deletePreviews(OutputInterface $output, bool $dryMode): void {
+		$previewsToDeleteCount = 0;
+
+		foreach ($this->getPreviewsToDelete() as ['name' => $previewFileId, 'path' => $filePath]) {
+			$output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE);
+
+			$previewsToDeleteCount++;
+
+			if ($dryMode) {
+				continue;
+			}
+
+			try {
+				$preview = $this->previewFolder->getFolder((string)$previewFileId);
+				$preview->delete();
+			} catch (NotFoundException $e) {
+				// continue
+			} catch (NotPermittedException $e) {
+				// continue
+			}
+		}
+
+		$output->writeln('Deleted ' . $previewsToDeleteCount . ' previews');
+	}
+
+	// Copy pasted and adjusted from
+	// "lib/private/Preview/BackgroundCleanupJob.php".
+	private function getPreviewsToDelete(): \Iterator {
+		$qb = $this->connection->getQueryBuilder();
+		$qb->select('path', 'mimetype')
+			->from('filecache')
+			->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId())));
+		$cursor = $qb->execute();
+		$data = $cursor->fetch();
+		$cursor->closeCursor();
+
+		if ($data === null) {
+			return [];
+		}
+
+		/*
+		 * This lovely like is the result of the way the new previews are stored
+		 * We take the md5 of the name (fileid) and split the first 7 chars. That way
+		 * there are not a gazillion files in the root of the preview appdata.
+		 */
+		$like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%';
+
+		$qb = $this->connection->getQueryBuilder();
+		$qb->select('a.name', 'b.path')
+			->from('filecache', 'a')
+			->leftJoin('a', 'filecache', 'b', $qb->expr()->eq(
+				$qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid'
+			))
+			->where(
+				$qb->expr()->andX(
+					$qb->expr()->like('a.path', $qb->createNamedParameter($like)),
+					$qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))),
+					$qb->expr()->orX(
+						$qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))),
+						$qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))),
+						$qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown')))
+					)
+				)
+			);
+
+		$cursor = $qb->execute();
+
+		while ($row = $cursor->fetch()) {
+			yield $row;
+		}
+
+		$cursor->closeCursor();
+	}
+}