sw:media:migrate zu wenig Speicher

Searching for all media files in your filesystem. This might take some time, depending on the number of media files you have.

Fatal error: Allowed memory size of 536870912 bytes exhausted (tried to allocate 20480 bytes) in /var/www/xxx/htdocs/vendor/league/flysystem/src/Adapter/Local.php on line 472

Wir haben ca. 200 GB an Bilddaten im image Ordner. Ja ich habe vorher das cleanup durchgeführt… :-/

Gibt es die Möglichkeit das trotzdem über sw:media:migrate abzuwickeln? Denkt Ihr memory_limit von 4G oder 8G reicht (aktuell 512MB)?

Eine --skip-scan Option gibt es hier nicht, richtig?

 

 

Gerade gesehen, dass es mit der neuen 5.3 Version scheinbar die skip-scan Option gibt, wer das ganze in SW 5.2 umsetzen möchte kann für die Migration einfach kurz die Files entsprechend anpassen (für SW 5.2.22):

Bei der ganzen Sache wird einfach durch die versch. Ordner durchiteriert, image mit dem wahrscheinlich größten Inhalt bekommt dafür eine eigenen foreach. Beim migrate CLI Befehl einfach --big oder -b mitgeben.

engine/Shopware/Bundle/MediaBundle/Commands/ImageMigrateCommand.php

 

setName('sw:media:migrate')
            ->setDescription('Migrate images to new structure')
            ->addOption('from', null, InputOption::VALUE_OPTIONAL)
            ->addOption('to', null, InputOption::VALUE_OPTIONAL)
            ->addOption('big', 'b', InputOption::VALUE_NONE, 'Uses iteration on directories to avoid crash on huge media folders');
    }

    /**
     * {@inheritdoc}
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
        $from = $input->getOption('from') ?: 'local';
        $to = $input->getOption('to') ?: 'local';

        $filesystemFactory = $this->getContainer()->get('shopware_media.media_service_factory');
        $fromFileSystem = $filesystemFactory->factory($from);
        $toFileSystem = $filesystemFactory->factory($to);
        $mediaMigration = $this->getContainer()->get('shopware_media.media_migration');

        if ($input->getOption('big')) {
            $output->writeln("sw:media:migrate will iterate through all your directories in your media folder. This might take a while.");

            $image_folders = glob("media/image/" . '*', GLOB_ONLYDIR );
            $media_folders = array_diff(glob("media/" . '*', GLOB_ONLYDIR ), ["media/image"]);

            foreach ($image_folders as $image_folder) $mediaMigration->migrate($fromFileSystem, $toFileSystem, $output, $image_folder);
            foreach ($media_folders as $media_folder)$mediaMigration->migrate($fromFileSystem, $toFileSystem, $output, $media_folder);

        } else $mediaMigration->migrate($fromFileSystem, $toFileSystem, $output, 'media');

    }

}

engine/Shopware/Bundle/MediaBundle/MediaMigration.php

 

 0,
        'skipped' => 0,
        'moved' => 0
    ];

    /**
     * Batch migration
     *
     * @param MediaServiceInterface $fromFilesystem
     * @param MediaServiceInterface $toFileSystem
     * @param OutputInterface $output
     * @param $folder_path
     * @throws \Exception
     */
    public function migrate(MediaServiceInterface $fromFilesystem, MediaServiceInterface $toFileSystem, OutputInterface $output, $folder_path)
    {
        if($folder_path == 'media') $output->writeln("Searching for all media files in your filesystem. This might take some time, depending on the number of media files you have.");
        else $output->writeln("Folder " . $folder_path . " is being processed.");

        foreach ($fromFilesystem->listFiles($folder_path) as $path) {
            $this->migrateFile($path, $fromFilesystem, $toFileSystem, $output);
        }

        $status = join('. ', array_map(
            function ($v, $k) {
                return $v." ".$k;
            },
            $this->counter,
            array_keys($this->counter)
        ));

        $output->writeln("Job done. ".$status);
    }

    /**
     * Migrate a single file
     *
     * @param string $path
     * @param MediaServiceInterface $fromFilesystem
     * @param MediaServiceInterface $toFileSystem
     * @param OutputInterface $output
     * @throws \Exception
     */
    private function migrateFile($path, MediaServiceInterface $fromFilesystem, MediaServiceInterface $toFileSystem, OutputInterface $output)
    {
        // only do migration if it's on the local filesystem since could take a long time
        // to read and write all the files
        if ($fromFilesystem->getAdapterType() === 'local') {
            if (!$fromFilesystem->isEncoded($path)) {
                $this->counter['migrated']++;
                $output->writeln("Migrate: ".$path);
                $fromFilesystem->migrateFile($path);
            }
        }

        // file already exists
        if ($toFileSystem->has($path)) {
            $this->counter['skipped']++;
            $output->writeln("SKIP: ".$path);
            return;
        }

        // move file to new filesystem and remove the old one
        if ($fromFilesystem->has($path)) {
            $this->counter['moved']++;
            $output->writeln("Move: ".$path);
            $success = $this->writeStream($toFileSystem, $path, $fromFilesystem->readStream($path));
            if ($success) {
                $fromFilesystem->delete($path);
            }

            return;
        }

        throw new \Exception("File not found: ".$path);
    }

    /**
     * @param MediaServiceInterface $toFileSystem
     * @param string $path
     * @param resource $contents
     * @return boolean
     */
    private function writeStream(MediaServiceInterface $toFileSystem, $path, $contents)
    {
        $path = $toFileSystem->encode($path);

        $dirString = '';
        $dirs = explode('/', dirname($path));
        foreach ($dirs as $dir) {
            $dirString .= '/' . $dir;
            $toFileSystem->createDir($dirString);
        }

        $toFileSystem->writeStream($path, $contents);

        return $toFileSystem->has($path);
    }
}