[stable34] fix: add files_metadata_index to the filecache partition

apps/dav/lib/Connector/Sabre/FilesPlugin.php

@@ -658,8 +658,7 @@ private function handleUpdatePropertiesMetadata(PropPatch $propPatch, Node $node
 				$mutation,
 				function (mixed $value) use ($accessRight, $knownMetadata, $node, $mutation, $filesMetadataManager): bool {
 					/** @var FilesMetadata $metadata */
-					$metadata = $filesMetadataManager->getMetadata((int)$node->getFileId(), true);
-					$metadata->setStorageId($node->getNode()->getStorage()->getCache()->getNumericStorageId());
+					$metadata = $filesMetadataManager->getMetadataForNode($node->getNode(), true);
 					$metadataKey = substr($mutation, strlen(self::FILE_METADATA_PREFIX));
 
 					// confirm metadata key is editable via PROPPATCH

apps/files/lib/Listener/SyncLivePhotosListener.php

@@ -61,9 +61,9 @@ public function handle(Event $event): void {
 			$peerFileId = null;
 
 			if ($event instanceof BeforeNodeRenamedEvent) {
-				$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getSource()->getId());
+				$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getSource());
 			} elseif ($event instanceof BeforeNodeDeletedEvent) {
-				$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getNode()->getId());
+				$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getNode());
 			} elseif ($event instanceof CacheEntriesRemovedEvent) {
 				$this->handleCacheEntriesRemovedEvent($event);
 			}
@@ -183,13 +183,11 @@ private function handleCopy(File $sourceFile, File $targetFile, File $peerFile):
 		}
 
 		/** @var FilesMetadata $targetMetadata */
-		$targetMetadata = $this->filesMetadataManager->getMetadata($targetFile->getId(), true);
-		$targetMetadata->setStorageId($targetFile->getStorage()->getCache()->getNumericStorageId());
+		$targetMetadata = $this->filesMetadataManager->getMetadataForNode($targetFile, true);
 		$targetMetadata->setString('files-live-photo', (string)$targetPeerFile->getId());
 		$this->filesMetadataManager->saveMetadata($targetMetadata);
 		/** @var FilesMetadata $peerMetadata */
-		$peerMetadata = $this->filesMetadataManager->getMetadata($targetPeerFile->getId(), true);
-		$peerMetadata->setStorageId($targetPeerFile->getStorage()->getCache()->getNumericStorageId());
+		$peerMetadata = $this->filesMetadataManager->getMetadataForNode($targetPeerFile, true);
 		$peerMetadata->setString('files-live-photo', (string)$targetFile->getId());
 		$this->filesMetadataManager->saveMetadata($peerMetadata);
 	}
@@ -250,7 +248,7 @@ private function handleCopyRecursive(Event $event, Node $sourceNode, Node $targe
 				return;
 			}
 
-			$peerFileId = $this->livePhotosService->getLivePhotoPeerId($sourceNode->getId());
+			$peerFileId = $this->livePhotosService->getLivePhotoPeerId($sourceNode);
 			if ($peerFileId === null) {
 				return;
 			}

apps/files/lib/Service/LivePhotosService.php

@@ -8,6 +8,7 @@
 
 namespace OCA\Files\Service;
 
+use OCP\Files\Node;
 use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
 use OCP\FilesMetadata\IFilesMetadataManager;
 
@@ -20,9 +21,9 @@ public function __construct(
 	/**
 	 * Get the associated live photo for a given file id
 	 */
-	public function getLivePhotoPeerId(int $fileId): ?int {
+	public function getLivePhotoPeerId(Node $file): ?int {
 		try {
-			$metadata = $this->filesMetadataManager->getMetadata($fileId);
+			$metadata = $this->filesMetadataManager->getMetadataForNode($file);
 		} catch (FilesMetadataNotFoundException $ex) {
 			return null;
 		}

apps/files_trashbin/lib/Listeners/SyncLivePhotosListener.php

@@ -40,7 +40,7 @@ public function handle(Event $event): void {
 		}
 
 		/** @var BeforeNodeRestoredEvent $event */
-		$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getSource()->getId());
+		$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getSource());
 
 		if ($peerFileId === null) {
 			return; // Not a live photo.

core/BackgroundJobs/GenerateMetadataJob.php

@@ -106,7 +106,7 @@ private function scanFolder(Folder $folder): void {
 			}
 
 			try {
-				$this->filesMetadataManager->getMetadata($node->getId(), false);
+				$this->filesMetadataManager->getMetadataForNode($node, false);
 			} catch (FilesMetadataNotFoundException) {
 				try {
 					$this->filesMetadataManager->refreshMetadata(

core/Command/FilesMetadata/Get.php

@@ -80,8 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 			}
 		}
 
+		$node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getFirstNodeById($fileId);
 		if ($input->getOption('refresh')) {
-			$node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getFirstNodeById($fileId);
 			if (!$node) {
 				throw new NotFoundException();
 			}
@@ -90,7 +90,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 				IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND
 			);
 		} else {
-			$metadata = $this->filesMetadataManager->getMetadata($fileId);
+			$metadata = $this->filesMetadataManager->getMetadataForNode($node);
 		}
 
 		if ($input->getOption('as-array')) {

lib/private/DB/Connection.php

@@ -94,6 +94,7 @@ class Connection extends PrimaryReadReplicaConnection {
 			'companion_tables' => [
 				'filecache_extended',
 				'files_metadata',
+				'files_metadata_index',
 			],
 			'primary_key' => 'fileid',
 			'shard_key' => 'storage',

lib/private/FilesMetadata/FilesMetadataManager.php

@@ -81,11 +81,10 @@ public function refreshMetadata(
 		$storageId = $node->getStorage()->getCache()->getNumericStorageId();
 		try {
 			/** @var FilesMetadata $metadata */
-			$metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId());
+			$metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId(), $node->getMountPoint()->getNumericStorageId());
 		} catch (FilesMetadataNotFoundException) {
-			$metadata = new FilesMetadata($node->getId());
+			$metadata = new FilesMetadata($node->getId(), $node->getMountPoint()->getNumericStorageId());
 		}
-		$metadata->setStorageId($storageId);
 
 		// if $process is LIVE, we enforce LIVE
 		// if $process is NAMED, we go NAMED
@@ -117,25 +116,36 @@ public function refreshMetadata(
 	/**
 	 * @param int $fileId file id
 	 * @param boolean $generate Generate if metadata does not exists
+	 * @param ?int $storageId
 	 *
 	 * @inheritDoc
 	 * @return IFilesMetadata
 	 * @throws FilesMetadataNotFoundException if not found
 	 * @since 28.0.0
 	 */
 	#[\Override]
-	public function getMetadata(int $fileId, bool $generate = false): IFilesMetadata {
+	public function getMetadata(int $fileId, bool $generate = false, ?int $storageId = null): IFilesMetadata {
+		if ($storageId === null) {
+			$storageId = $this->metadataRequestService->getStorageIdForFileId($fileId);
+		}
 		try {
-			return $this->metadataRequestService->getMetadataFromFileId($fileId);
+			return $this->metadataRequestService->getMetadataFromFileId($fileId, $storageId);
 		} catch (FilesMetadataNotFoundException $ex) {
 			if ($generate) {
-				return new FilesMetadata($fileId);
+				return new FilesMetadata($fileId, $storageId);
 			}
 
 			throw $ex;
 		}
 	}
 
+	#[\Override]
+	public function getMetadataForNode(Node $node, bool $generate = false): IFilesMetadata {
+		/** @var FilesMetadata $metadata */
+		$metadata = $this->getMetadata($node->getId(), $generate, $node->getMountPoint()->getNumericStorageId());
+		return $metadata;
+	}
+
 	/**
 	 * returns metadata of multiple file ids
 	 *
@@ -232,7 +242,7 @@ public function deleteMetadataForFiles(int $storage, array $fileIds): void {
 		}
 
 		try {
-			$this->indexRequestService->dropIndexForFiles($fileIds);
+			$this->indexRequestService->dropIndexForFiles($storage, $fileIds);
 		} catch (DBException $e) {
 			$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]);
 		}

lib/private/FilesMetadata/Model/FilesMetadata.php

@@ -27,10 +27,10 @@ class FilesMetadata implements IFilesMetadata {
 	private bool $updated = false;
 	private int $lastUpdate = 0;
 	private string $syncToken = '';
-	private ?int $storageId = null;
 
 	public function __construct(
 		private int $fileId = 0,
+		private int $storageId = 0,
 	) {
 	}
 
@@ -44,22 +44,11 @@ public function getFileId(): int {
 		return $this->fileId;
 	}
 
-	public function getStorageId(): ?int {
+	#[\Override]
+	public function getStorageId(): int {
 		return $this->storageId;
 	}
 
-	/**
-	 * Set which storage the file this metadata belongs to.
-	 *
-	 * This helps with sharded filecache setups to know where to store the metadata
-	 *
-	 * @param int $storageId
-	 * @return void
-	 */
-	public function setStorageId(int $storageId): void {
-		$this->storageId = $storageId;
-	}
-
 	/**
 	 * @inheritDoc
 	 * @return int timestamp

lib/private/FilesMetadata/Service/IndexRequestService.php

@@ -39,6 +39,7 @@ public function __construct(
 	 */
 	public function updateIndex(IFilesMetadata $filesMetadata, string $key): void {
 		$fileId = $filesMetadata->getFileId();
+		$storageId = $filesMetadata->getStorageId();
 		try {
 			$metadataType = $filesMetadata->getType($key);
 		} catch (FilesMetadataNotFoundException $e) {
@@ -56,11 +57,11 @@ public function updateIndex(IFilesMetadata $filesMetadata, string $key): void {
 		try {
 			$this->dropIndex($fileId, $key);
 			match ($metadataType) {
-				IMetadataValueWrapper::TYPE_STRING => $this->insertIndexString($fileId, $key, $filesMetadata->getString($key)),
-				IMetadataValueWrapper::TYPE_INT => $this->insertIndexInt($fileId, $key, $filesMetadata->getInt($key)),
-				IMetadataValueWrapper::TYPE_BOOL => $this->insertIndexBool($fileId, $key, $filesMetadata->getBool($key)),
-				IMetadataValueWrapper::TYPE_STRING_LIST => $this->insertIndexStringList($fileId, $key, $filesMetadata->getStringList($key)),
-				IMetadataValueWrapper::TYPE_INT_LIST => $this->insertIndexIntList($fileId, $key, $filesMetadata->getIntList($key))
+				IMetadataValueWrapper::TYPE_STRING => $this->insertIndexString($fileId, $storageId, $key, $filesMetadata->getString($key)),
+				IMetadataValueWrapper::TYPE_INT => $this->insertIndexInt($fileId, $storageId, $key, $filesMetadata->getInt($key)),
+				IMetadataValueWrapper::TYPE_BOOL => $this->insertIndexBool($fileId, $storageId, $key, $filesMetadata->getBool($key)),
+				IMetadataValueWrapper::TYPE_STRING_LIST => $this->insertIndexStringList($fileId, $storageId, $key, $filesMetadata->getStringList($key)),
+				IMetadataValueWrapper::TYPE_INT_LIST => $this->insertIndexIntList($fileId, $storageId, $key, $filesMetadata->getIntList($key))
 			};
 		} catch (FilesMetadataNotFoundException|FilesMetadataTypeException|DbException $e) {
 			$this->dbConnection->rollBack();
@@ -79,12 +80,13 @@ public function updateIndex(IFilesMetadata $filesMetadata, string $key): void {
 	 *
 	 * @throws DbException
 	 */
-	private function insertIndexString(int $fileId, string $key, string $value): void {
+	private function insertIndexString(int $fileId, int $storageId, string $key, string $value): void {
 		$qb = $this->dbConnection->getQueryBuilder();
 		$qb->insert(self::TABLE_METADATA_INDEX)
 			->setValue('meta_key', $qb->createNamedParameter($key))
 			->setValue('meta_value_string', $qb->createNamedParameter($value))
-			->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT));
+			->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))
+			->hintShardKey('storage', $storageId);
 		$qb->executeStatement();
 	}
 
@@ -97,12 +99,13 @@ private function insertIndexString(int $fileId, string $key, string $value): voi
 	 *
 	 * @throws DbException
 	 */
-	public function insertIndexInt(int $fileId, string $key, int $value): void {
+	public function insertIndexInt(int $fileId, int $storageId, string $key, int $value): void {
 		$qb = $this->dbConnection->getQueryBuilder();
 		$qb->insert(self::TABLE_METADATA_INDEX)
 			->setValue('meta_key', $qb->createNamedParameter($key))
 			->setValue('meta_value_int', $qb->createNamedParameter($value, IQueryBuilder::PARAM_INT))
-			->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT));
+			->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))
+			->hintShardKey('storage', $storageId);
 		$qb->executeStatement();
 	}
 
@@ -115,12 +118,13 @@ public function insertIndexInt(int $fileId, string $key, int $value): void {
 	 *
 	 * @throws DbException
 	 */
-	public function insertIndexBool(int $fileId, string $key, bool $value): void {
+	public function insertIndexBool(int $fileId, int $storageId, string $key, bool $value): void {
 		$qb = $this->dbConnection->getQueryBuilder();
 		$qb->insert(self::TABLE_METADATA_INDEX)
 			->setValue('meta_key', $qb->createNamedParameter($key))
 			->setValue('meta_value_int', $qb->createNamedParameter(($value) ? '1' : '0', IQueryBuilder::PARAM_INT))
-			->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT));
+			->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))
+			->hintShardKey('storage', $storageId);
 		$qb->executeStatement();
 	}
 
@@ -133,9 +137,9 @@ public function insertIndexBool(int $fileId, string $key, bool $value): void {
 	 *
 	 * @throws DbException
 	 */
-	public function insertIndexStringList(int $fileId, string $key, array $values): void {
+	public function insertIndexStringList(int $fileId, int $storageId, string $key, array $values): void {
 		foreach ($values as $value) {
-			$this->insertIndexString($fileId, $key, $value);
+			$this->insertIndexString($fileId, $storageId, $key, $value);
 		}
 	}
 
@@ -148,9 +152,9 @@ public function insertIndexStringList(int $fileId, string $key, array $values):
 	 *
 	 * @throws DbException
 	 */
-	public function insertIndexIntList(int $fileId, string $key, array $values): void {
+	public function insertIndexIntList(int $fileId, int $storageId, string $key, array $values): void {
 		foreach ($values as $value) {
-			$this->insertIndexInt($fileId, $key, $value);
+			$this->insertIndexInt($fileId, $storageId, $key, $value);
 		}
 	}
 
@@ -185,14 +189,15 @@ public function dropIndex(int $fileId, string $key = ''): void {
 	 *
 	 * @throws DbException
 	 */
-	public function dropIndexForFiles(array $fileIds, string $key = ''): void {
+	public function dropIndexForFiles(int $storageId, array $fileIds, string $key = ''): void {
 		$chunks = array_chunk($fileIds, 1000);
 
 		foreach ($chunks as $chunk) {
 			$qb = $this->dbConnection->getQueryBuilder();
 			$expr = $qb->expr();
 			$qb->delete(self::TABLE_METADATA_INDEX)
-				->where($expr->in('file_id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)));
+				->where($expr->in('file_id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
+				->hintShardKey('storage', $storageId);
 
 			if ($key !== '') {
 				$qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key)));

lib/private/FilesMetadata/Service/MetadataRequestService.php

@@ -28,25 +28,12 @@ public function __construct(
 	) {
 	}
 
-	private function getStorageId(IFilesMetadata $filesMetadata): int {
-		if ($filesMetadata instanceof FilesMetadata) {
-			$storage = $filesMetadata->getStorageId();
-			if ($storage) {
-				return $storage;
-			}
-		}
-		// all code paths that lead to saving metadata *should* have the storage id set
-		// this fallback is there just in case
+	public function getStorageIdForFileId(int $fileId): int {
 		$query = $this->dbConnection->getQueryBuilder();
 		$query->select('storage')
 			->from('filecache')
-			->where($query->expr()->eq('fileid', $query->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)));
-		$storageId = $query->executeQuery()->fetchColumn();
-
-		if ($filesMetadata instanceof FilesMetadata) {
-			$filesMetadata->setStorageId($storageId);
-		}
-		return $storageId;
+			->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
+		return (int)$query->executeQuery()->fetchOne();
 	}
 
 	/**
@@ -59,7 +46,7 @@ private function getStorageId(IFilesMetadata $filesMetadata): int {
 	public function store(IFilesMetadata $filesMetadata): void {
 		$qb = $this->dbConnection->getQueryBuilder();
 		$qb->insert(self::TABLE_METADATA)
-			->hintShardKey('storage', $this->getStorageId($filesMetadata))
+			->hintShardKey('storage', $filesMetadata->getStorageId())
 			->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))
 			->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize())))
 			->setValue('sync_token', $qb->createNamedParameter($this->generateSyncToken()))
@@ -75,11 +62,12 @@ public function store(IFilesMetadata $filesMetadata): void {
 	 * @return IFilesMetadata
 	 * @throws FilesMetadataNotFoundException if no metadata are found in database
 	 */
-	public function getMetadataFromFileId(int $fileId): IFilesMetadata {
+	public function getMetadataFromFileId(int $fileId, int $storageId): IFilesMetadata {
 		try {
 			$qb = $this->dbConnection->getQueryBuilder();
 			$qb->select('json', 'sync_token')->from(self::TABLE_METADATA);
-			$qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
+			$qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)))
+				->hintShardKey('storage', $storageId);
 			$result = $qb->executeQuery();
 			$data = $result->fetch();
 			$result->closeCursor();
@@ -91,8 +79,7 @@ public function getMetadataFromFileId(int $fileId): IFilesMetadata {
 		if ($data === false) {
 			throw new FilesMetadataNotFoundException();
 		}
-
-		$metadata = new FilesMetadata($fileId);
+		$metadata = new FilesMetadata($fileId, $storageId);
 		$metadata->importFromDatabase($data);
 
 		return $metadata;
@@ -174,7 +161,7 @@ public function updateMetadata(IFilesMetadata $filesMetadata): int {
 		$expr = $qb->expr();
 
 		$qb->update(self::TABLE_METADATA)
-			->hintShardKey('files_metadata', $this->getStorageId($filesMetadata))
+			->hintShardKey('files_metadata', $filesMetadata->getStorageId())
 			->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize())))
 			->set('sync_token', $qb->createNamedParameter($this->generateSyncToken()))
 			->set('last_update', $qb->func()->now())