Compare commits

..

No commits in common. 'a6fcc6d77f9dd63ca42eb73f2d12893e65a614ab' and '396387ec7aeb4b616f7c3f0418bfaa1856431265' have entirely different histories.

  1. 34
      README.md
  2. 8
      drush.services.yml
  3. 4
      islandora_inplace_media.info.yml
  4. 16
      islandora_inplace_media.services.yml
  5. 109
      src/Commands/IslandoraInplaceMediaCommands.php
  6. 35
      src/Plugin/QueueWorker/InplaceMediaQueueWorker.php
  7. 74
      src/Service/InplaceMediaProcessor.php

34
README.md

@ -1,36 +1,6 @@
# Islandora In-Place Media ## INTRODUCTION
A Drush-driven ingestion tool for Islandora that creates Media entities
from files already on disk — **without moving them**.
Designed for **large-scale, resumable, idempotent, parallel-safe** ingest
workflows.
---
## ✨ Features
* ✅ In-place ingestion (no file copying unless paths differ)
* ✅ Idempotent (safe to re-run; no duplicate Media)
* ✅ Automatic resume after failure
* ✅ Drush command interface
* ✅ Queue-based background processing
* ✅ Concurrency-safe sharding for parallel workers
* ✅ Explicit ownership (uid = 1, configurable later)
* ✅ Works with cron or manual queue runners
---
## 📦 Requirements
* Drupal 10 or 11
* Drush 10+
* Islandora Media bundles already configured
* Files already present on disk (e.g. `public://`, `private://`, or mounted paths)
---
The Islandora Inplace Media module allows users to create and attach media from files staged on the server.
## INSTALLATION ## INSTALLATION

8
drush.services.yml

@ -1,8 +0,0 @@
services:
islandora_inplace_media.commands:
class: Drupal\islandora_inplace_media\Commands\IslandoraInplaceMediaCommands
arguments:
- '@islandora_inplace_media.processor'
- '@state'
tags:
- { name: drush.command }

4
islandora_inplace_media.info.yml

@ -1,7 +1,7 @@
name: 'Islandora Inplace Media' name: 'Islandora Inplace Media'
type: module type: module
description: 'Allows Uploaded files to be attached to existing nodes as media' description: 'Allows Uploaded files to be attched to exisitng nodes as media'
package: Islandora package: Custom
core_version_requirement: ^10 || ^11 core_version_requirement: ^10 || ^11
dependencies: dependencies:
- islandora:islandora - islandora:islandora

16
islandora_inplace_media.services.yml

@ -1,14 +1,4 @@
services: services:
islandora_inplace_media.utils:
logger.channel.islandora_inplace_media: class: Drupal\islandora_inplace_media\Utils
class: Drupal\Core\Logger\LoggerChannel arguments: ['@entity_type.manager', '@file_system', '@logger.factory', '@entity_field.manager']
factory: logger.factory:get
arguments: ['islandora_inplace_media']
islandora_inplace_media.processor:
class: Drupal\islandora_inplace_media\Service\InplaceMediaProcessor
arguments:
- '@file_system'
- '@file.repository'
- '@logger.channel.islandora_inplace_media'
- '@entity_type.manager'

109
src/Commands/IslandoraInplaceMediaCommands.php

@ -1,109 +0,0 @@
<?php
namespace Drupal\islandora_inplace_media\Commands;
use Drush\Commands\DrushCommands;
use Drupal\islandora_inplace_media\Service\InplaceMediaProcessor;
use Drupal\Core\State\StateInterface;
use Symfony\Component\Console\Helper\ProgressBar;
class IslandoraInplaceMediaCommands extends DrushCommands {
protected InplaceMediaProcessor $processor;
protected StateInterface $state;
public function __construct(
InplaceMediaProcessor $processor,
StateInterface $state,
) {
parent::__construct();
$this->processor = $processor;
$this->state = $state;
}
/**
* Create Islandora media from files.
*
* @command islandora:inplace-media
* @aliases iim
*
* @option source_dir
* Source directory containing files.
* @option destination_path
* Destination directory (public://, private://, or absolute).
* @option media_type
* Media bundle machine name.
* @option media_use
* Taxonomy term ID for field_media_use.
* @option file_type
* Media file field name.
* @option limit
* Maximum number of files to process.
* @option offset
* Number of files to skip before processing.
* @option queue
* Queue files instead of processing immediately.
* @option shards
* Total number of queue shards.
*/
public function run(array $options = [
'source_dir' => NULL,
'destination_path' => NULL,
'media_type' => NULL,
'media_use' => NULL,
'file_type' => 'field_media_file',
'limit' => NULL,
'offset' => 0,
'queue' => FALSE,
'shards' => 1,
]) {
$files = array_values(array_diff(
scandir($options['source_dir']),
['.', '..']
));
$job_id = hash('sha256', serialize([
$options['source_dir'],
$options['destination_path'],
$options['media_type'],
$options['media_use'],
$options['file_type'],
]));
$state_key = "islandora_inplace_media.completed.$job_id";
$completed = $this->state->get($state_key, []);
$files = array_values(array_diff($files, $completed));
if ($options['queue']) {
foreach ($files as $file) {
$shard = crc32($file) % (int) $options['shards'];
\Drupal::queue('islandora_inplace_media')
->createItem([
'file' => $file,
'options' => $options,
'job_id' => $job_id,
'shard' => $shard,
]);
}
$this->output()->writeln('Files queued with sharding.');
return;
}
$progress = new ProgressBar($this->output(), count($files));
$progress->start();
foreach ($files as $file) {
if ($this->processor->processFile($file, $options)) {
$completed[] = $file;
$this->state->set($state_key, $completed);
}
$progress->advance();
}
$progress->finish();
$this->output()->writeln('');
}
}

35
src/Plugin/QueueWorker/InplaceMediaQueueWorker.php

@ -1,35 +0,0 @@
<?php
namespace Drupal\islandora_inplace_media\Plugin\QueueWorker;
use Drupal\Core\Queue\QueueWorkerBase;
/**
* @QueueWorker(
* id = "islandora_inplace_media",
* title = @Translation("Islandora Inplace Media"),
* cron = {"time" = 60}
* )
*/
class InplaceMediaQueueWorker extends QueueWorkerBase {
public function processItem($data) {
// Optional shard filtering (used by drush queue:run --shard).
if (isset($data['shard']) && isset($_SERVER['DRUSH_SHARD'])) {
if ((int) $_SERVER['DRUSH_SHARD'] !== (int) $data['shard']) {
return;
}
}
$processor = \Drupal::service('islandora_inplace_media.processor');
$state = \Drupal::state();
if ($processor->processFile($data['file'], $data['options'])) {
$key = "islandora_inplace_media.completed.{$data['job_id']}";
$completed = $state->get($key, []);
$completed[] = $data['file'];
$state->set($key, $completed);
}
}
}

74
src/Service/InplaceMediaProcessor.php

@ -1,74 +0,0 @@
<?php
namespace Drupal\islandora_inplace_media\Service;
use Drupal\Core\File\FileSystemInterface;
use Drupal\file\FileRepositoryInterface;
use Drupal\Core\File\FileExists;
use Drupal\file\Entity\File;
use Drupal\media\Entity\Media;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Psr\Log\LoggerInterface;
class InplaceMediaProcessor {
protected $mediaStorage;
public function __construct(
FileSystemInterface $fileSystem,
FileRepositoryInterface $fileRepository,
LoggerInterface $logger,
EntityTypeManagerInterface $entityTypeManager,
) {
$this->fileSystem = $fileSystem;
$this->fileRepository = $fileRepository;
$this->logger = $logger;
$this->mediaStorage = $entityTypeManager->getStorage('media');
}
protected function mediaExists(string $uri, array $opts): bool {
return (bool) $this->mediaStorage->getQuery()
->condition('bundle', $opts['media_type'])
->condition("{$opts['file_type']}.entity.uri", $uri)
->condition('field_media_use.target_id', $opts['media_use'])
->accessCheck(FALSE)
->range(0, 1)
->execute();
}
public function processFile(string $file, array $opts): bool {
$source = "{$opts['source_dir']}/{$file}";
$dest = "{$opts['destination_path']}/{$file}";
if (!file_exists($source)) {
$this->logger->warning('Missing file @file', ['@file' => $source]);
return FALSE;
}
$uri = ($source === $dest)
? $dest
: $this->fileSystem->copy($source, $dest, FileExists::Rename);
if ($this->mediaExists($uri, $opts)) {
return TRUE;
}
$fileEntity = $this->fileRepository->loadByUri($uri)
?? File::create(['uri' => $uri, 'status' => 1]);
$fileEntity->save();
preg_match('/^(\d+)_/', $file, $m);
$nid = $m[1] ?? NULL;
Media::create([
'bundle' => $opts['media_type'],
'name' => $file,
$opts['file_type'] => ['target_id' => $fileEntity->id()],
'field_media_use' => ['target_id' => $opts['media_use']],
'field_media_of' => $nid,
])->save();
return TRUE;
}
}
Loading…
Cancel
Save