@ -6,7 +6,10 @@ namespace Drupal\ocfl_export;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\File\FileSystemInterface;
use Drupal\Core\KeyValueStore\KeyValueFactoryInterface;
use Drupal\Core\KeyValueStore\KeyValueStoreInterface;
use Drupal\Core\Logger\LoggerChannelFactoryInterface;
use Drupal\Core\Session\AccountProxyInterface;
use Psr\Log\LoggerInterface;
/**
@ -14,14 +17,45 @@ use Psr\Log\LoggerInterface;
*/
class OcflStorageService {
/**
* Logical path of the metadata JSON file within every object.
*/
public const METADATA_FILE = 'node.json';
/**
* The OCFL storage layout extension used for the storage root.
*/
public const LAYOUT_EXTENSION = '0003-hash-and-id-n-tuple-storage-layout';
/**
* Configuration of the storage layout extension.
*/
protected const LAYOUT_CONFIG = [
'extensionName' => self::LAYOUT_EXTENSION,
'digestAlgorithm' => 'sha256',
'tupleSize' => 3,
'numberOfTuples' => 3,
];
/**
* The logger channel.
*/
protected LoggerInterface $logger;
/**
* Cache of file digests keyed by URI, to avoid rehashing unchanged files.
*/
protected KeyValueStoreInterface $digestCache;
public function __construct(
protected ConfigFactoryInterface $configFactory,
protected FileSystemInterface $fileSystem,
LoggerChannelFactoryInterface $loggerFactory,
KeyValueFactoryInterface $keyValueFactory,
protected AccountProxyInterface $currentUser,
) {
$this->logger = $loggerFactory->get('ocfl_export');
$this->digestCache = $keyValueFactory->get('ocfl_export.digest_cache');
}
/**
@ -57,7 +91,7 @@ class OcflStorageService {
// Create OCFL root declaration file.
$declarationFile = $root . '/0=ocfl_1.1';
if (!file_exists($declarationFile)) {
$content = "ocfl_1.1\nhttps://ocfl.io/1.1/spec/#root-conformance-declaration\n ";
$content = "ocfl_1.1\n";
if (file_put_contents($declarationFile, $content) === FALSE) {
$this->logger->error('Failed to create OCFL root declaration file.');
return FALSE;
@ -68,110 +102,357 @@ class OcflStorageService {
$layoutFile = $root . '/ocfl_layout.json';
if (!file_exists($layoutFile)) {
$layout = [
'extension' => '0002-flat-direct-storage-layout' ,
'description' => 'OCFL storage using flat direct layout with UUID s.',
'extension' => self::LAYOUT_EXTENSION ,
'description' => 'Hashed n-tuple layout (sha256, 3 tuples of 3 characters) ending in the percent-encoded object ID; object IDs are urn:uuid URI s.',
];
if (file_put_contents($layoutFile, json_encode($layout, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)) === FALSE) {
$this->logger->error('Failed to create OCFL layout file.');
return FALSE;
}
}
else {
// Never write objects into a root that declares a different layout.
$layout = json_decode((string) file_get_contents($layoutFile), TRUE);
if (($layout['extension'] ?? NULL) !== self::LAYOUT_EXTENSION) {
$this->logger->error('OCFL storage root @path uses layout @layout, but this module requires @required.', [
'@path' => $root,
'@layout' => $layout['extension'] ?? 'unknown',
'@required' => self::LAYOUT_EXTENSION,
]);
return FALSE;
}
}
// Create the layout extension's configuration file.
$extensionDir = $root . '/extensions/' . self::LAYOUT_EXTENSION;
if (!file_exists($extensionDir . '/config.json')) {
if (!is_dir($extensionDir) & & !$this->fileSystem->mkdir($extensionDir, 0755, TRUE)) {
$this->logger->error('Failed to create OCFL extension directory: @path', ['@path' => $extensionDir]);
return FALSE;
}
if (file_put_contents($extensionDir . '/config.json', json_encode(self::LAYOUT_CONFIG, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)) === FALSE) {
$this->logger->error('Failed to create OCFL layout extension config.');
return FALSE;
}
}
return TRUE;
}
/**
* Save or update an OCFL object.
* Build the OCFL object ID for an entity UUID.
*
* @param string $uuid
* The entity UUID.
*
* @return string
* The object ID, a urn:uuid URI.
*/
public function getObjectId(string $uuid): string {
return 'urn:uuid:' . $uuid;
}
/**
* Get the directory of an object.
*
* @param string $uuid
* The entity UUID.
*
* @return string
* The absolute path to the object directory.
*/
public function getObjectPath(string $uuid): string {
return $this->getStorageRoot() . '/' . $this->getObjectRelativePath($this->getObjectId($uuid));
}
/**
* Map an object ID to its path using the hash-and-id n-tuple layout.
*
* The path is the first tuples of the ID's digest followed by the
* percent-encoded ID, e.g.
* "3f2/a9c/e01/urn%3auuid%3a11111111-2222-3333-4444-555555555555".
*
* @param string $objectId
* The object ID (UUID).
* The object ID.
*
* @return string
* The object path relative to the storage root.
*
* @see https://ocfl.github.io/extensions/0003-hash-and-id-n-tuple-storage-layout.html
*/
public function getObjectRelativePath(string $objectId): string {
$config = self::LAYOUT_CONFIG;
$digest = hash($config['digestAlgorithm'], $objectId);
// Percent-encode every byte except [A-Za-z0-9_-], using lowercase hex.
$encoded = preg_replace_callback(
'/[^A-Za-z0-9_-]/',
fn (array $match) => '%' . bin2hex($match[0]),
$objectId,
);
if (strlen($encoded) > 100) {
$encoded = substr($encoded, 0, 100) . '-' . $digest;
}
$tuples = str_split(substr($digest, 0, $config['tupleSize'] * $config['numberOfTuples']), $config['tupleSize']);
return implode('/', $tuples) . '/' . $encoded;
}
/**
* Save or update an OCFL object.
*
* Each call creates a new version. Content files whose digest already
* appears in the object's manifest are not copied again; the new version's
* state simply references the existing content path (forward-delta
* deduplication as described in the OCFL spec).
*
* @param string $uuid
* The entity UUID; the object ID is derived from it.
* @param array $data
* The data to save.
* The metadata to save as node.json.
* @param array $files
* Binary files to include, keyed by logical path with the source file URI
* (any readable stream wrapper URI or local path) as the value.
* @param string|null $message
* Optional version message.
*
* @return bool
* TRUE if save was successful.
*/
public function saveObject(string $objectId, array $data): bool {
public function saveObject(string $uuid, array $data, array $files = [], ?string $message = NULL ): bool {
if (!$this->initializeStorageRoot()) {
return FALSE;
}
$root = $this->getStorageRoot();
$objectPath = $root . '/' . $objectId;
$objectId = $this->getObjectId($uuid );
$objectPath = $this->getObjectPath($uuid) ;
// Check if object exists and get current version.
$currentVersion = $this->getCurrentVersion($objectPath);
$existing = $this->loadInventory($objectPath);
$currentVersion = $existing ? (int) substr($existing['head'], 1) : 0 ;
$newVersion = $currentVersion + 1;
$versionDir = 'v' . $newVersion;
$versionPath = $objectPath . '/' . $versionDir;
// Create object directory structure.
$contentPath = $objectPath . '/' . $versionDir . '/content';
if (!is_dir($contentPath)) {
if (!$this->fileSystem->mkdir($contentPath, 0755, TRUE)) {
$this->logger->error('Failed to create OCFL object content directory: @path', ['@path' => $contentPath]);
return FALSE;
}
// A version directory that is not yet recorded in the inventory is left
// over from an interrupted export; its files would not be in the manifest.
if (is_dir($versionPath)) {
$this->fileSystem->deleteRecursive($versionPath);
}
if (!$this->fileSystem->mkdir($versionPath . '/content', 0755, TRUE)) {
$this->logger->error('Failed to create OCFL object content directory: @path', ['@path' => $versionPath . '/content']);
return FALSE;
}
// Create object declaration file for new objects.
if ($currentVersion === 0) {
$declarationFile = $objectPath . '/0=ocfl_object_1.1';
$content = "ocfl_object_1.1\nhttps://ocfl.io/1.1/spec/#object-conformance-declaration\n";
$content = "ocfl_object_1.1\n";
if (file_put_contents($declarationFile, $content) === FALSE) {
$this->logger->error('Failed to create OCFL object declaration file.');
return FALSE;
}
}
// Write node.json to content directory.
$nodeJsonPath = $contentPath . '/node.json';
$manifest = $existing['manifest'] ?? [];
$state = [];
// Add binary files first so missing ones can be recorded in node.json.
$missing = [];
foreach ($files as $logicalPath => $sourceUri) {
$digest = $this->addContentFile($objectPath, $versionDir, (string) $logicalPath, $sourceUri, $manifest);
if ($digest === NULL) {
$missing[] = (string) $logicalPath;
continue;
}
$state[$digest][] = (string) $logicalPath;
}
if ($missing) {
$data['_metadata']['missing_files'] = $missing;
}
$jsonContent = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
if (file_put_contents($nodeJsonPath, $jsonContent) === FALSE) {
$this->logger->error('Failed to write node.json to OCFL object.');
if ($jsonContent === FALSE) {
$this->logger->error('Failed to enc ode me tadata f or OCFL object @id .', ['@id' => $objectId] );
return FALSE;
}
$digest = hash('sha512', $jsonContent);
if (!isset($manifest[$digest])) {
$contentPath = $versionDir . '/content/' . self::METADATA_FILE;
if (file_put_contents($objectPath . '/' . $contentPath, $jsonContent) === FALSE) {
$this->logger->error('Failed to write @file to OCFL object.', ['@file' => self::METADATA_FILE]);
return FALSE;
}
$manifest[$digest] = [$contentPath];
}
$state[$digest][] = self::METADATA_FILE;
// Generate and write inventory.
$inventory = $this->generateInventory($objectId, $objectPath, $newVersion, $nodeJsonPath);
$versions = $existing['versions'] ?? [];
$versions[$versionDir] = [
'created' => date('c'),
'message' => $message ?? ($newVersion === 1 ? 'Initial version' : 'Updated version'),
'state' => $state,
'user' => [
'name' => $this->currentUser->isAnonymous() ? 'Drupal system' : $this->currentUser->getDisplayName(),
],
];
// Write inventory to version directory.
$versionInventoryPath = $objectPath . '/' . $versionDir . '/inventory.json';
$inventoryJson = json_encode($inventory, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
if (file_put_contents($versionInventoryPath, $inventoryJson) === FALSE) {
$this->logger->error('Failed to write version inventory.json.');
return FALSE;
}
// Don't leave an empty content directory when everything was deduplicated.
@rmdir($versionPath . '/content');
// Write inventory checksum to version directory.
$versionChecksumPath = $objectPath . '/' . $versionDir . '/inventory.json.sha512';
$checksum = hash('sha512', $inventoryJson);
if (file_put_contents($versionChecksumPath, $checksum . ' inventory.json') === FALSE) {
$this->logger->error('Failed to write version inventory checksum.');
return FALSE;
}
$inventory = [
'id' => $objectId,
'type' => 'https://ocfl.io/1.1/spec/#inventory',
'digestAlgorithm' => 'sha512',
'head' => $versionDir,
'contentDirectory' => 'content',
'manifest' => $manifest,
'versions' => $versions,
];
// Write inventory to object root.
$rootInventoryPath = $objectPath . '/inventory.json';
if (file_put_contents($rootInventoryPath, $inventoryJson) === FALSE) {
$this->logger->error('Failed to write root inventory.json.');
$inventoryJson = json_encode($inventory, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
$sidecar = hash('sha512', $inventoryJson) . " inventory.json\n";
// Write the version inventory, then replace the root inventory last so
// the object only advances to the new head once the version is complete.
if (file_put_contents($versionPath . '/inventory.json', $inventoryJson) === FALSE
|| file_put_contents($versionPath . '/inventory.json.sha512', $sidecar) === FALSE) {
$this->logger->error('Failed to write version inventory for OCFL object @id.', ['@id' => $objectId]);
return FALSE;
}
// Write inventory checksum to object root.
$rootChecksumPath = $objectPath . '/inventory.json.sha512';
if (file_put_contents($rootChecksumPath, $checksum . ' inventory.json' ) === FALSE) {
$this->logger->error('Failed to write root inventory checksum.' );
$tmpInventory = $objectPath . '/inventory.json.tmp';
if (file_put_contents($tmpInventory, $inventoryJson) === FALSE
|| !rename($tmpInventory, $objectPath . '/inventory.json')
|| file_put_contents($objectPath . '/inventory.json.sha512', $sidecar ) === FALSE) {
$this->logger->error('Failed to write root inventory for OCFL object @id.', ['@id' => $objectId] );
return FALSE;
}
$this->logger->info('OCFL object @id saved as version @version.', [
$this->logger->info('OCFL object @id saved as version @version (@count files) .', [
'@id' => $objectId,
'@version' => $versionDir,
'@count' => count($files) - count($missing) + 1,
]);
return TRUE;
}
/**
* Add a binary file to a version, deduplicating against the manifest.
*
* @param string $objectPath
* The object root path.
* @param string $versionDir
* The version directory name (e.g. "v3").
* @param string $logicalPath
* The logical path of the file within the object.
* @param string $sourceUri
* The URI of the source file.
* @param array $manifest
* The manifest, updated in place when the file is copied.
*
* @return string|null
* The file's digest, or NULL if it could not be read or copied.
*/
protected function addContentFile(string $objectPath, string $versionDir, string $logicalPath, string $sourceUri, array & $manifest): ?string {
$digest = $this->getDigest($sourceUri);
if ($digest === NULL) {
$this->logger->warning('Could not read @uri for OCFL object path @path; skipping.', [
'@uri' => $sourceUri,
'@path' => $logicalPath,
]);
return NULL;
}
if (isset($manifest[$digest])) {
return $digest;
}
$contentPath = $versionDir . '/content/' . $logicalPath;
$destination = $objectPath . '/' . $contentPath;
$dir = dirname($destination);
if (!is_dir($dir) & & !$this->fileSystem->mkdir($dir, 0755, TRUE)) {
$this->logger->error('Failed to create directory @dir.', ['@dir' => $dir]);
return NULL;
}
if (!@copy($sourceUri, $destination) || hash_file('sha512', $destination) !== $digest) {
$this->logger->error('Failed to copy @uri into OCFL object at @path, or the copy failed fixity verification.', [
'@uri' => $sourceUri,
'@path' => $contentPath,
]);
@unlink($destination);
return NULL;
}
$manifest[$digest] = [$contentPath];
return $digest;
}
/**
* Compute the sha512 digest of a file, using a size/mtime keyed cache.
*
* @param string $uri
* The file URI.
*
* @return string|null
* The hex digest, or NULL if the file can't be read.
*/
protected function getDigest(string $uri): ?string {
clearstatcache(TRUE, $uri);
$stat = @stat($uri);
$cacheable = $stat !== FALSE & & !empty($stat['mtime']);
$key = hash('sha256', $uri);
if ($cacheable) {
$cached = $this->digestCache->get($key);
if (is_array($cached)
& & $cached['uri'] === $uri
& & $cached['size'] === (int) $stat['size']
& & $cached['mtime'] === (int) $stat['mtime']) {
return $cached['digest'];
}
}
$digest = @hash_file('sha512', $uri);
if ($digest === FALSE) {
return NULL;
}
if ($cacheable) {
$this->digestCache->set($key, [
'uri' => $uri,
'size' => (int) $stat['size'],
'mtime' => (int) $stat['mtime'],
'digest' => $digest,
]);
}
return $digest;
}
/**
* Load and minimally validate an object's root inventory.
*
* @param string $objectPath
* The path to the object directory.
*
* @return array|null
* The inventory, or NULL if the object doesn't exist yet.
*/
protected function loadInventory(string $objectPath): ?array {
$inventoryPath = $objectPath . '/inventory.json';
if (!file_exists($inventoryPath)) {
return NULL;
}
$inventory = json_decode(file_get_contents($inventoryPath), TRUE);
if (!is_array($inventory) || !isset($inventory['head'])) {
return NULL;
}
return $inventory;
}
/**
* Mark an OCFL object as deleted (for nodes).
*
@ -182,8 +463,7 @@ class OcflStorageService {
* TRUE if marking was successful.
*/
public function markDeleted(\Drupal\node\NodeInterface $node): bool {
$root = $this->getStorageRoot();
$objectPath = $root . '/' . $node->uuid();
$objectPath = $this->getObjectPath($node->uuid());
if (!is_dir($objectPath)) {
return TRUE;
@ -192,6 +472,7 @@ class OcflStorageService {
// Create a new version with deletion marker.
$data = [
'uuid' => $node->uuid(),
'nid' => (int) $node->id(),
'type' => $node->bundle(),
'title' => $node->getTitle(),
'deleted' => TRUE,
@ -205,7 +486,7 @@ class OcflStorageService {
],
];
return $this->saveObject($node->uuid(), $data);
return $this->saveObject($node->uuid(), $data, [], 'Node deleted' );
}
/**
@ -218,8 +499,7 @@ class OcflStorageService {
* TRUE if marking was successful.
*/
public function markMediaDeleted(\Drupal\media\MediaInterface $media): bool {
$root = $this->getStorageRoot();
$objectPath = $root . '/' . $media->uuid();
$objectPath = $this->getObjectPath($media->uuid());
if (!is_dir($objectPath)) {
return TRUE;
@ -228,6 +508,7 @@ class OcflStorageService {
// Create a new version with deletion marker.
$data = [
'uuid' => $media->uuid(),
'mid' => (int) $media->id(),
'type' => $media->bundle(),
'name' => $media->getName(),
'deleted' => TRUE,
@ -241,98 +522,7 @@ class OcflStorageService {
],
];
return $this->saveObject($media->uuid(), $data);
}
/**
* Get the current version number for an object.
*
* @param string $objectPath
* The path to the object directory.
*
* @return int
* The current version number (0 if object doesn't exist).
*/
protected function getCurrentVersion(string $objectPath): int {
if (!is_dir($objectPath)) {
return 0;
}
$inventoryPath = $objectPath . '/inventory.json';
if (!file_exists($inventoryPath)) {
return 0;
}
$inventory = json_decode(file_get_contents($inventoryPath), TRUE);
if (!$inventory || !isset($inventory['head'])) {
return 0;
}
// Parse version number from head (e.g., "v3" -> 3).
return (int) substr($inventory['head'], 1);
}
/**
* Generate OCFL inventory for an object.
*
* @param string $objectId
* The object ID.
* @param string $objectPath
* The object path.
* @param int $version
* The new version number.
* @param string $nodeJsonPath
* The path to the new node.json file.
*
* @return array
* The inventory data.
*/
protected function generateInventory(string $objectId, string $objectPath, int $version, string $nodeJsonPath): array {
$versionDir = 'v' . $version;
// Calculate checksum for the new content file.
$contentChecksum = hash_file('sha512', $nodeJsonPath);
// Load existing inventory if this is an update.
$existingManifest = [];
$existingVersions = [];
if ($version > 1) {
$existingInventoryPath = $objectPath . '/inventory.json';
if (file_exists($existingInventoryPath)) {
$existing = json_decode(file_get_contents($existingInventoryPath), TRUE);
$existingManifest = $existing['manifest'] ?? [];
$existingVersions = $existing['versions'] ?? [];
}
}
// Build manifest - maps checksums to file paths.
$manifest = $existingManifest;
$contentPath = $versionDir . '/content/node.json';
$manifest[$contentChecksum] = [$contentPath];
// Build version state - maps checksums to logical paths.
$versionState = [
$contentChecksum => ['node.json'],
];
// Build versions array.
$versions = $existingVersions;
$versions[$versionDir] = [
'created' => date('c'),
'message' => $version === 1 ? 'Initial version' : 'Updated version',
'state' => $versionState,
];
$inventory = [
'id' => $objectId,
'type' => 'https://ocfl.io/1.1/spec/#inventory',
'digestAlgorithm' => 'sha512',
'head' => $versionDir,
'manifest' => $manifest,
'versions' => $versions,
];
return $inventory;
return $this->saveObject($media->uuid(), $data, [], 'Media deleted');
}
/**
@ -343,7 +533,9 @@ class OcflStorageService {
*/
public function isInitialized(): bool {
$root = $this->getStorageRoot();
return file_exists($root . '/0=ocfl_1.1') & & file_exists($root . '/ocfl_layout.json');
return file_exists($root . '/0=ocfl_1.1')
& & file_exists($root . '/ocfl_layout.json')
& & file_exists($root . '/extensions/' . self::LAYOUT_EXTENSION . '/config.json');
}
/**
@ -356,15 +548,35 @@ class OcflStorageService {
* Object info or NULL if not found.
*/
public function getObjectInfo(string $uuid): ?array {
$root = $this->getStorageRoot();
$objectPath = $root . '/' . $uuid;
$inventoryPath = $objectPath . '/inventory.json';
return $this->loadInventory($this->getObjectPath($uuid));
}
if (!file_exists($inventoryPath)) {
return NULL;
/**
* Resolve a logical path in the head version to its file on disk.
*
* Because unchanged content is deduplicated, the file may live in an
* earlier version's content directory.
*
* @param string $uuid
* The object UUID.
* @param string $logicalPath
* The logical path within the object.
*
* @return string|null
* The absolute path, or NULL if not found.
*/
public function resolveHeadPath(string $uuid, string $logicalPath): ?string {
$inventory = $this->getObjectInfo($uuid);
$state = $inventory['versions'][$inventory['head'] ?? '']['state'] ?? [];
foreach ($state as $digest => $paths) {
if (in_array($logicalPath, $paths, TRUE) & & isset($inventory['manifest'][$digest][0])) {
$filePath = $this->getObjectPath($uuid) . '/' . $inventory['manifest'][$digest][0];
return file_exists($filePath) ? $filePath : NULL;
}
}
return json_decode(file_get_contents($inventoryPath), TRUE);
return NULL ;
}
/**
@ -377,19 +589,7 @@ class OcflStorageService {
* The absolute path to the latest node.json file, or NULL if not found.
*/
public function getLatestExportPath(string $uuid): ?string {
$objectInfo = $this->getObjectInfo($uuid);
if (!$objectInfo || !isset($objectInfo['head'])) {
return NULL;
}
$root = $this->getStorageRoot();
$filePath = $root . '/' . $uuid . '/' . $objectInfo['head'] . '/content/node.json';
if (!file_exists($filePath)) {
return NULL;
}
return $filePath;
return $this->resolveHeadPath($uuid, self::METADATA_FILE);
}
/**
@ -403,24 +603,17 @@ class OcflStorageService {
*/
public function getExportStatus(string $uuid): ?array {
$objectInfo = $this->getObjectInfo($uuid);
if (!$objectInfo || !isset($objectInfo['head'])) {
$filePath = $this->getLatestExportPath($uuid);
if (!$objectInfo || !$filePath) {
return NULL;
}
$head = $objectInfo['head'];
$root = $this->getStorageRoot();
$filePath = $root . '/' . $uuid . '/' . $head . '/content/node.json';
if (!file_exists($filePath)) {
return NULL;
}
$created = $objectInfo['versions'][$head]['created'] ?? NULL;
return [
'version' => $head,
'path' => $filePath,
'created' => $created,
'created' => $objectInfo['versions'][$head]['created'] ?? NULL,
];
}
