TASK: 7.3 upmerges

Author: dlubitz

Neos.Flow/Classes/Http/Helper/ResponseInformationHelper.php

@@ -228,7 +228,7 @@ public static function makeStandardsCompliant(ResponseInterface $response, Reque
             }
         }
 
-        if (!$response->hasHeader('Content-Length')) {
+        if (!$response->hasHeader('Content-Length') && $response->getBody()->getSize() !== null) {
             $response = $response->withHeader('Content-Length', $response->getBody()->getSize());
         }
 

Neos.Flow/Classes/Mvc/ActionResponse.php

@@ -347,6 +347,7 @@ public function buildHttpResponse(): ResponseInterface
      */
     private function hasContent(): bool
     {
-        return $this->content->getSize() > 0;
+        $contentSize = $this->content->getSize();
+        return $contentSize === null || $contentSize > 0;
     }
 }

Neos.Flow/Classes/ResourceManagement/ResourceManager.php

@@ -11,12 +11,10 @@
  * source code.
  */
 
-use enshrined\svgSanitize\Sanitizer;
 use Neos\Flow\Annotations as Flow;
 use Neos\Flow\Log\Utility\LogEnvironment;
 use Neos\Flow\ObjectManagement\ObjectManagerInterface;
 use Neos\Flow\Persistence\PersistenceManagerInterface;
-use Neos\Utility\MediaTypes;
 use Neos\Utility\ObjectAccess;
 use Neos\Flow\ResourceManagement\Storage\StorageInterface;
 use Neos\Flow\ResourceManagement\Storage\WritableStorageInterface;
@@ -162,30 +160,14 @@ public function importResource($source, $collectionName = ResourceManager::DEFAU
         $collection = $this->collections[$collectionName];
 
         try {
-            if (is_resource($source)) {
-                $mediaType = MediaTypes::getMediaTypeFromResource($source);
-                if ($this->isSanitizingRequired($mediaType)) {
-                    $content = stream_get_contents($source);
-                    $resource = $this->importResourceFromContent($content, '', $collectionName, $forcedPersistenceObjectIdentifier);
-                } else {
-                    $resource = $collection->importResource($source);
-                }
-            } else {
-                $resource = fopen($source, 'rb');
-                $mediaType = MediaTypes::getMediaTypeFromResource($resource);
-                fclose($resource);
-                if ($this->isSanitizingRequired($mediaType)) {
-                    $content = file_get_contents($source);
-                    $resource = $this->importResourceFromContent($content, '', $collectionName, $forcedPersistenceObjectIdentifier);
-                } else {
-                    $resource = $collection->importResource($source);
-                }
-                $pathInfo = UnicodeFunctions::pathinfo($source);
-                $resource->setFilename($pathInfo['basename']);
-            }
+            $resource = $collection->importResource($source);
             if ($forcedPersistenceObjectIdentifier !== null) {
                 ObjectAccess::setProperty($resource, 'Persistence_Object_Identifier', $forcedPersistenceObjectIdentifier, true);
             }
+            if (!is_resource($source)) {
+                $pathInfo = UnicodeFunctions::pathinfo($source);
+                $resource->setFilename($pathInfo['basename']);
+            }
         } catch (Exception $exception) {
             throw new Exception(sprintf('Importing a file into the resource collection "%s" failed: %s', $collectionName, $exception->getMessage()), 1375197120, $exception);
         }
@@ -224,11 +206,6 @@ public function importResourceFromContent($content, $filename, $collectionName =
             throw new Exception(sprintf('Tried to import a file into the resource collection "%s" but no such collection exists. Please check your settings and the code which triggered the import.', $collectionName), 1380878131);
         }
 
-        $mediaType = MediaTypes::getMediaTypeFromFileContent($content);
-        if ($this->isSanitizingRequired($mediaType)) {
-            $content = $this->sanitizeImportedFileContent($mediaType, $content, $filename);
-        }
-
         /* @var CollectionInterface $collection */
         $collection = $this->collections[$collectionName];
 
@@ -631,44 +608,6 @@ protected function initializeCollections()
         }
     }
 
-    /**
-     * Decide weather the given media-type has to be sanitized
-     * for now this only checks svg file to solve the issue here https://nvd.nist.gov/vuln/detail/CVE-2023-37611
-     *
-     * @todo create a feature from this and allow to register code for sanitizing file content before importing
-     */
-    protected function isSanitizingRequired(string $mediaType): bool
-    {
-        return $mediaType === 'image/svg+xml';
-    }
-
-    /**
-     * Sanitize file content and remove content that is suspicious
-     * for now this only checks svg file to solve the issue here https://nvd.nist.gov/vuln/detail/CVE-2023-37611
-     *
-     * @todo create a feature from this and allow to register code for sanitizing file content before importing
-     */
-    protected function sanitizeImportedFileContent(string $mediaType, string $content, $filename = ''): string
-    {
-        if ($mediaType === 'image/svg+xml') {
-            // @todo: Simplify again when https://github.com/darylldoyle/svg-sanitizer/pull/90 is merged and released.
-            $previousXmlErrorHandling = libxml_use_internal_errors(true);
-            $sanitizer = new Sanitizer();
-            $sanitizedContent = $sanitizer->sanitize($content);
-            libxml_clear_errors();
-            libxml_use_internal_errors($previousXmlErrorHandling);
-            $issues = $sanitizer->getXmlIssues();
-            if ($issues && count($issues) > 0) {
-                if ($sanitizedContent === false) {
-                    throw new Exception('Sanitizing of suspicious file "' . $filename . '" failed during import.', 1695395560);
-                }
-                $content = $sanitizedContent;
-                $this->logger->warning(sprintf('Imported file "%s" contained suspicious content and was sanitized.', $filename), $issues);
-            }
-        }
-        return $content;
-    }
-
     /**
      * Prepare an uploaded file to be imported as resource object. Will check the validity of the file,
      * move it outside of upload folder if open_basedir is enabled and check the filename.

Neos.Flow/Documentation/TheDefinitiveGuide/PartV/ChangeLogs/7317.rst

@@ -0,0 +1,159 @@
+`7.3.17 (2023-11-22) <https://github.com/neos/flow-development-collection/releases/tag/7.3.17>`_
+================================================================================================
+
+Overview of merged pull requests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`BUGFIX: Set InvalidHashException status code to 400 <https://github.com/neos/flow-development-collection/pull/3234>`_
+----------------------------------------------------------------------------------------------------------------------
+
+``InvalidHashException`` now declares ``400`` as it's status code (not the inherited ``500`` it has now), as that is clearly a case of a "bad request".
+
+* See: `#3159 <https://github.com/neos/flow-development-collection/issues/3159>`_
+
+**Upgrade instructions**
+
+This might need adjustment, if you rely on the ``InvalidHashException`` throwing a status code of ``500`` somewhere.
+
+
+* Packages: ``Flow``
+
+`BUGFIX: Return the expected result for `is_dir('resource://sha1')` <https://github.com/neos/flow-development-collection/pull/3226>`_
+-------------------------------------------------------------------------------------------------------------------------------------
+
+* Fixes: `#3225 <https://github.com/neos/flow-development-collection/issues/3225>`_
+
+
+* Packages: ``Flow``
+
+`BUGFIX: Use method to set validated instances container <https://github.com/neos/flow-development-collection/pull/3210>`_
+--------------------------------------------------------------------------------------------------------------------------
+
+* Fixes: `#3205 <https://github.com/neos/flow-development-collection/issues/3205>`_
+
+
+* Packages: ``Flow``
+
+`BUGFIX: Require collection packages as `self.version` again <https://github.com/neos/flow-development-collection/pull/3206>`_
+------------------------------------------------------------------------------------------------------------------------------
+
+* See: `#3035 <https://github.com/neos/flow-development-collection/issues/3035>`_ for the original change
+
+
+* Packages: ``Flow`` ``Eel`` ``FluidAdaptor`` ``Kickstarter``
+
+`BUGFIX: Only set distinct on count clause if explicitely set to improve performance <https://github.com/neos/flow-development-collection/pull/3140>`_
+------------------------------------------------------------------------------------------------------------------------------------------------------
+
+F.e. Postgres has performance issues with large datasets and the DISTINCT clause. In a test this change reduced the query time of a count query for ~900.000 entities by >80%.
+
+In a custom project this affected their Neos Media.UI in which the following results were found:
+
+* Count all assets |  580ms ->  260ms
+* Query 20 assets  |  690ms ->  350ms
+* Query 100 assets |  990ms -> 650ms
+* Module load | 1900ms -> 1400ms
+
+**Review instructions**
+
+Everything should work the same, as https://github.com/neos/flow-development-collection/pull/415 already sets the distinct flag where (possibly) necessary.
+
+
+* Packages: ``Flow``
+
+`BUGFIX: Sanitize uploaded svg files from suspicious content <https://github.com/neos/flow-development-collection/pull/3172>`_
+------------------------------------------------------------------------------------------------------------------------------
+
+Adding an internal methods ``isSanitizingRequired`` and ``sanitizeImportedFileContent`` to the resourceManager. The import is adjusted to first determine the mediaType of an imported resource to decide wether sanitizing is needed which for now happens only for SVG files. If no sanitizing is needed the code will perform as before by passing streams or filenames around.
+
+If suspicious content was removed from a warning is logged that mentions the remove data and line. The sanitizing is done using "enshrined/svg-sanitize" that is used by other cms aswell.
+
+The initial implementation will only sanitize SVG files as those can contain malicious scripts. In future this should be expanded to a feature that allows registering of custom sanitizing functions.
+
+The sanitizing logic itself ist basically the same as what is done by typo3 here: https://github.com/TYPO3/typo3/blob/`357b07064cf2c7f1735cfb8f73ac4a7248ab040e <https://github.com/neos/flow-development-collection/commit/357b07064cf2c7f1735cfb8f73ac4a7248ab040e>`_/typo3/sysext/core/Classes/Resource/Security/SvgSanitizer.php
+
+This addresses the issue described here: https://nvd.nist.gov/vuln/detail/CVE-2023-37611
+
+**Review Instructions**
+
+The change adds quite a bit of complexity to the importResource method to avoid loading the file content into ram whenever possible. As this method accepts filenames and resources this leads to quite some nested checking. I consider this kindoff necessary as one does not want to read a full video file into php ram to check wether it may be an svg. 
+
+Better suggestions are welcome.
+
+
+* Packages: ``Utility.MediaTypes``
+
+`TASK: Update default .htaccess for _Resources <https://github.com/neos/flow-development-collection/pull/3238>`_
+----------------------------------------------------------------------------------------------------------------
+
+PHP 5 is a thing of the past, but for PHP 8 the module is name just ``mod_php.c``, so that needs to be added.
+
+**Upgrade instructions**
+
+Depending in the way you deploy and whether you have that file even in version control, the change might need to be applied manually to your setup.
+
+**Review instructions**
+
+
+* Packages: ``Flow``
+
+`TASK: Routing Documentation Adjustment  <https://github.com/neos/flow-development-collection/pull/3231>`_
+----------------------------------------------------------------------------------------------------------
+
+Correction of an erroneous path in routing documentation.
+
+* Packages: ``Flow``
+
+`TASK: PEG Parser declares properties <https://github.com/neos/flow-development-collection/pull/3215>`_
+-------------------------------------------------------------------------------------------------------
+
+Prevents deprecation warnings for dynamic properties.
+
+* Packages: ``Flow`` ``Eel``
+
+`TASK: Clean up stored throwable dumps <https://github.com/neos/flow-development-collection/pull/3187>`_
+--------------------------------------------------------------------------------------------------------
+
+Whenever a new dump is written, check the existing dumps and remove those that are older than allowed or exceed the maximum count.
+
+By default nothing is cleaned up.
+
+* Resolves: `#3158 <https://github.com/neos/flow-development-collection/issues/3158>`_
+
+**Review instructions**
+
+Should remove old dump files as configured…
+
+
+* Packages: ``Flow``
+
+`TASK: Fix overlooked dependency… <https://github.com/neos/flow-development-collection/pull/3207>`_
+-----------------------------------------------------------------------------------------------------
+
+* See: `#3035 <https://github.com/neos/flow-development-collection/issues/3035>`_ for the original change
+
+
+* Packages: ``Flow``
+
+`TASK: Fix cache RedisBackend unittest <https://github.com/neos/flow-development-collection/pull/3196>`_
+--------------------------------------------------------------------------------------------------------
+
+A test failed due to a missing return value from a method not being mocked (correctly),
+
+
+* Packages: ``Cache``
+
+`TASK: Fix documentation builds <https://github.com/neos/flow-development-collection/pull/3195>`_
+-------------------------------------------------------------------------------------------------
+
+… by pinning updated dependencies.
+
+**Review instructions**
+
+Best is to see if the builds succeed on RTD again with this merged…
+
+
+* Packages: ``Flow``
+
+`Detailed log <https://github.com/neos/flow-development-collection/compare/7.3.16...7.3.17>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Neos.Flow/Documentation/requirements.txt

@@ -58,5 +58,5 @@ sphinxcontrib-qthelp==1.0.6
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.9
     # via sphinx
-urllib3==2.0.6
+urllib3==2.0.7
     # via requests

Neos.Flow/composer.json

@@ -51,8 +51,7 @@
 
         "composer/composer": "^2.2.8",
 
-        "egulias/email-validator": "^2.1.17 || ^3.0",
-        "enshrined/svg-sanitize": "^0.16.0"
+        "egulias/email-validator": "^2.1.17 || ^3.0"
     },
     "require-dev": {
         "mikey179/vfsstream": "^1.6.10",

Neos.Utility.MediaTypes/Classes/MediaTypes.php

@@ -1830,21 +1830,6 @@ public static function getMediaTypeFromFileContent(string $fileContent): string
         return isset(self::$mediaTypeToFileExtension[$mediaType]) ? $mediaType : 'application/octet-stream';
     }
 
-    /**
-     * Returns a Media Type based on the given resource
-     *
-     * @param resource $resource The resource to determine the media type from
-     * @return string The IANA Internet Media Type
-     */
-    public static function getMediaTypeFromResource($resource): string
-    {
-        if (!is_resource($resource)) {
-            throw new \TypeError('Argument "resource" has to be a resource');
-        }
-        $mediaType = self::trimMediaType(mime_content_type($resource));
-        return isset(self::$mediaTypeToFileExtension[$mediaType]) ? $mediaType : 'application/octet-stream';
-    }
-
     /**
      * Returns the primary filename extension based on the given Media Type.
      *

Neos.Utility.MediaTypes/Tests/Unit/MediaTypesTest.php

@@ -68,22 +68,6 @@ public function getMediaTypeFromFileContent(string $filename, string $expectedMe
         self::assertSame($expectedMediaType, MediaTypes::getMediaTypeFromFileContent($fileContent));
     }
 
-    /**
-     * @test
-     * @dataProvider filesAndMediaTypes
-     */
-    public function getMediaTypeFromResource(string $filename, string $expectedMediaType)
-    {
-        $filePath = __DIR__ . '/Fixtures/' . $filename;
-        $resource = is_file($filePath) ? fopen($filePath, 'rb') : fopen('data://text/plain,', 'rb');
-        if ($resource !== false) {
-            self::assertSame($expectedMediaType, MediaTypes::getMediaTypeFromResource($resource));
-            fclose($resource);
-        } else {
-            $this->fail('fixture ' . $filePath . ' could not be read');
-        }
-    }
-
     /**
      * Data Provider
      */

composer.json

@@ -33,7 +33,6 @@
         "neos/composer-plugin": "^2.0",
         "composer/composer": "^2.2.8",
         "egulias/email-validator": "^2.1.17 || ^3.0",
-        "enshrined/svg-sanitize": "^0.16.0",
         "typo3fluid/fluid": "~2.7.0",
         "guzzlehttp/psr7": "^1.8.4 || ^2.1.1",
         "ext-mbstring": "*"