Merge branch 'master' into feature-disable-indexin-per-nodetype-or-namespace

# Conflicts:
#	Classes/Mapping/NodeTypeMappingBuilder.php

Author: kdambekalns

.travis.yml

@@ -1,20 +1,21 @@
 language: php
+sudo: false
+git:
+  depth: 5
+  submodules: false
+addons:
+  apt:
+    packages:
+      - oracle-java8-set-default
 matrix:
   include:
-    - php: 7.0
-      env: ES=1
-    - php: 7.0
-      env: ES=2
-    - php: 7.1
-      env: ES=1
-    - php: 7.1
-      env: ES=2
-sudo: false
+    - php: 7.2
+      env: ES=5
+
 before_install:
-  - export NEOS_TARGET_VERSION=3.0
+  - export NEOS_TARGET_VERSION=4.0
   - cd ..
-  - if [ "$ES" = 1 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.6.zip && unzip elasticsearch-1.7.6.zip && mv elasticsearch-1.7.6 elasticsearch; fi
-  - if [ "$ES" = 2 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.4.5.zip && unzip elasticsearch-2.4.5.zip && mv elasticsearch-2.4.5 elasticsearch; fi
+  - if [ "$ES" = 5 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.zip && unzip elasticsearch-5.6.8.zip && mv elasticsearch-5.6.8 elasticsearch; fi
   - cd elasticsearch
   - bin/elasticsearch -d
   - cd ..
@@ -29,5 +30,4 @@ install:
   - cd neos-base-distribution
 script:
   - bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit
-  - if [ "$ES" = 1 ]; then FLOW_CONTEXT="Testing/ElasticVersion1" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
-  - if [ "$ES" = 2 ]; then FLOW_CONTEXT="Testing/ElasticVersion2" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
+  - if [ "$ES" = 5 ]; then FLOW_CONTEXT="Testing/ElasticVersion5" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi

Classes/Client/ClientFactory.php

@@ -32,6 +32,7 @@ class ClientFactory
      * Create a client
      *
      * @return Client
+     * @throws \Flowpack\ElasticSearch\Exception
      */
     public function create()
     {

Classes/Command/NodeIndexCommandController.php

@@ -12,8 +12,8 @@
  */
 
 use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\Error\ErrorInterface;
+use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
 use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface;
-use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder;
 use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Service\ErrorHandlingService;
 use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Service\IndexWorkspaceTrait;
 use Flowpack\ElasticSearch\Domain\Model\Mapping;
@@ -23,13 +23,14 @@
 use Neos\ContentRepository\Domain\Repository\NodeDataRepository;
 use Neos\ContentRepository\Domain\Repository\WorkspaceRepository;
 use Neos\ContentRepository\Domain\Service\ContentDimensionPresetSourceInterface;
-use Neos\ContentRepository\Domain\Service\Context;
-use Neos\ContentRepository\Domain\Service\ContextFactoryInterface;
 use Neos\ContentRepository\Search\Indexer\NodeIndexerInterface;
 use Neos\Flow\Annotations as Flow;
 use Neos\Flow\Cli\CommandController;
 use Neos\Flow\Configuration\ConfigurationManager;
+use Neos\Flow\Configuration\Exception\InvalidConfigurationTypeException;
+use Neos\Flow\Mvc\Exception\StopActionException;
 use Neos\Flow\ObjectManagement\ObjectManagerInterface;
+use Neos\Neos\Controller\CreateContentContextTrait;
 use Symfony\Component\Yaml\Yaml;
 
 /**
@@ -41,6 +42,8 @@ class NodeIndexCommandController extends CommandController
 {
     use IndexWorkspaceTrait;
 
+    use CreateContentContextTrait;
+
     /**
      * @Flow\Inject
      * @var ErrorHandlingService
@@ -79,7 +82,7 @@ class NodeIndexCommandController extends CommandController
 
     /**
      * @Flow\Inject
-     * @var NodeTypeMappingBuilder
+     * @var NodeTypeMappingBuilderInterface
      */
     protected $nodeTypeMappingBuilder;
 
@@ -95,12 +98,6 @@ class NodeIndexCommandController extends CommandController
      */
     protected $configurationManager;
 
-    /**
-     * @Flow\Inject
-     * @var ContextFactoryInterface
-     */
-    protected $contextFactory;
-
     /**
      * @var array
      */
@@ -110,6 +107,7 @@ class NodeIndexCommandController extends CommandController
      * Called by the Flow object framework after creating the object and resolving all dependencies.
      *
      * @param integer $cause Creation cause
+     * @throws InvalidConfigurationTypeException
      */
     public function initializeObject($cause)
     {
@@ -159,6 +157,7 @@ public function showMappingCommand()
      * @param string $identifier
      * @param string $workspace
      * @return void
+     * @throws StopActionException
      */
     public function indexNodeCommand($identifier, $workspace = null)
     {
@@ -226,8 +225,9 @@ public function indexNodeCommand($identifier, $workspace = null)
      * @param string $workspace name of the workspace which should be indexed
      * @param string $postfix Index postfix, index with the same postfix will be deleted if exist
      * @return void
+     * @throws StopActionException
      */
-    public function buildCommand($limit = null, $update = false, $workspace = null, $postfix = null)
+    public function buildCommand($limit = null, $update = false, $workspace = null, $postfix = '')
     {
         if ($workspace !== null && $this->workspaceRepository->findByIdentifier($workspace) === null) {
             $this->logger->log('The given workspace (' . $workspace . ') does not exist.', LOG_ERR);
@@ -303,33 +303,12 @@ public function cleanupCommand()
             }
         } catch (ApiException $exception) {
             $response = json_decode($exception->getResponse());
-            $this->logger->log(sprintf('Nothing removed. ElasticSearch responded with status %s, saying "%s: %s"', $response->status, $response->error->type, $response->error->reason));
-        }
-    }
-
-    /**
-     * Create a ContentContext based on the given workspace name
-     *
-     * @param string $workspaceName Name of the workspace to set for the context
-     * @param array $dimensions Optional list of dimensions and their values which should be set
-     * @return Context
-     */
-    protected function createContentContext($workspaceName, array $dimensions = [])
-    {
-        $contextProperties = [
-            'workspaceName' => $workspaceName,
-            'invisibleContentShown' => true,
-            'inaccessibleContentShown' => true
-        ];
-
-        if ($dimensions !== []) {
-            $contextProperties['dimensions'] = $dimensions;
-            $contextProperties['targetDimensions'] = array_map(function ($dimensionValues) {
-                return array_shift($dimensionValues);
-            }, $dimensions);
+            if ($response->error instanceof \stdClass) {
+                $this->logger->log(sprintf('Nothing removed. ElasticSearch responded with status %s, saying "%s: %s"', $response->status, $response->error->type, $response->error->reason));
+            } else {
+                $this->logger->log(sprintf('Nothing removed. ElasticSearch responded with status %s, saying "%s"', $response->status, $response->error));
+            }
         }
-
-        return $this->contextFactory->create($contextProperties);
     }
 
     /**
@@ -338,7 +317,7 @@ protected function createContentContext($workspaceName, array $dimensions = [])
      * @param string $postfix
      * @return void
      */
-    protected function createNewIndex($postfix)
+    protected function createNewIndex(string $postfix)
     {
         $this->nodeIndexer->setIndexNamePostfix($postfix ?: time());
         if ($this->nodeIndexer->getIndex()->exists() === true) {

Classes/Command/NodeTypeCommandController.php

@@ -11,6 +11,8 @@
  * source code.
  */
 
+use Neos\ContentRepository\Domain\Service\NodeTypeManager;
+use Neos\ContentRepository\Exception\NodeTypeNotFoundException;
 use Neos\Flow\Annotations as Flow;
 use Neos\Flow\Cli\CommandController;
 
@@ -25,7 +27,7 @@ class NodeTypeCommandController extends CommandController
 {
     /**
      * @Flow\Inject
-     * @var \Neos\ContentRepository\Domain\Service\NodeTypeManager
+     * @var NodeTypeManager
      */
     protected $nodeTypeManager;
 
@@ -34,6 +36,7 @@ class NodeTypeCommandController extends CommandController
      *
      * @param string $nodeType the node type to optionally filter for
      * @return void
+     * @throws NodeTypeNotFoundException
      */
     public function showCommand($nodeType = null)
     {

Classes/Driver/AbstractIndexerDriver.php

@@ -11,20 +11,27 @@
  * source code.
  */
 
+use Neos\Flow\Annotations as Flow;
 use Neos\ContentRepository\Domain\Model\NodeInterface;
 
 /**
  * Abstract Fulltext Indexer Driver
  */
 abstract class AbstractIndexerDriver extends AbstractDriver
 {
+    /**
+     * @Flow\Inject
+     * @var NodeTypeMappingBuilderInterface
+     */
+    protected $nodeTypeMappingBuilder;
+
     /**
      * Whether the node is configured as fulltext root.
      *
      * @param NodeInterface $node
-     * @return boolean
+     * @return bool
      */
-    protected function isFulltextRoot(NodeInterface $node)
+    protected function isFulltextRoot(NodeInterface $node): bool
     {
         if ($node->getNodeType()->hasConfiguration('search')) {
             $elasticSearchSettingsForNode = $node->getNodeType()->getConfiguration('search');
@@ -35,4 +42,26 @@ protected function isFulltextRoot(NodeInterface $node)
 
         return false;
     }
+
+    /**
+     * @param NodeInterface $node
+     * @return NodeInterface|null
+     */
+    protected function findClosestFulltextRoot(NodeInterface $node)
+    {
+        $closestFulltextNode = $node;
+        while (!$this->isFulltextRoot($closestFulltextNode)) {
+            $closestFulltextNode = $closestFulltextNode->getParent();
+            if ($closestFulltextNode === null) {
+                // root of hierarchy, no fulltext root found anymore, abort silently...
+                if ($node->getPath() !== '/' && $node->getPath() !== '/sites') {
+                    $this->logger->log(sprintf('NodeIndexer: No fulltext root found for node %s (%s)', $node->getIdentifier(), $node->getContextPath()), LOG_WARNING, null, 'ElasticSearch (CR)');
+                }
+
+                return null;
+            }
+        }
+
+        return $closestFulltextNode;
+    }
 }

Classes/Driver/AbstractNodeTypeMappingBuilder.php

@@ -0,0 +1,84 @@
+<?php
+namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver;
+
+/*
+ * This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
+ *
+ * (c) Contributors of the Neos Project - www.neos.io
+ *
+ * This package is Open Source Software. For the full copyright and license
+ * information, please view the LICENSE file which was distributed with this
+ * source code.
+ */
+
+use Neos\ContentRepository\Domain\Service\NodeTypeManager;
+use Neos\Error\Messages\Result;
+use Neos\Flow\Annotations as Flow;
+use Neos\Flow\Configuration\ConfigurationManager;
+use Neos\Flow\Configuration\Exception\InvalidConfigurationTypeException;
+use Neos\Flow\ObjectManagement\ObjectManagerInterface;
+
+/**
+ * Builds the mapping information for Content Repository Node Types in Elasticsearch
+ *
+ * @Flow\Scope("singleton")
+ */
+abstract class AbstractNodeTypeMappingBuilder implements NodeTypeMappingBuilderInterface
+{
+    /**
+     * The default configuration for a given property type in NodeTypes.yaml, if no explicit elasticSearch section defined there.
+     *
+     * @var array
+     */
+    protected $defaultConfigurationPerType;
+
+    /**
+     * @Flow\Inject
+     * @var NodeTypeManager
+     */
+    protected $nodeTypeManager;
+
+    /**
+     * @var Result
+     */
+    protected $lastMappingErrors;
+
+    /**
+     * @Flow\Inject
+     * @var ConfigurationManager
+     */
+    protected $configurationManager;
+
+    /**
+     * Called by the Flow object framework after creating the object and resolving all dependencies.
+     *
+     * @param integer $cause Creation cause
+     * @throws InvalidConfigurationTypeException
+     */
+    public function initializeObject($cause)
+    {
+        if ($cause === ObjectManagerInterface::INITIALIZATIONCAUSE_CREATED) {
+            $settings = $this->configurationManager->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepository.Search');
+            $this->defaultConfigurationPerType = $settings['defaultConfigurationPerType'];
+        }
+    }
+
+    /**
+     * Converts a Content Repository Node Type name into a name which can be used for an Elasticsearch Mapping
+     *
+     * @param string $nodeTypeName
+     * @return string
+     */
+    public function convertNodeTypeNameToMappingName(string $nodeTypeName): string
+    {
+        return str_replace('.', '-', $nodeTypeName);
+    }
+
+    /**
+     * @return Result
+     */
+    public function getLastMappingErrors()
+    {
+        return $this->lastMappingErrors;
+    }
+}

Classes/Driver/AbstractQuery.php

@@ -109,6 +109,22 @@ public function aggregation($name, array $aggregationDefinition, $parentPath = '
         }
     }
 
+    /**
+     * {@inheritdoc}
+     * @throws Exception\QueryBuildingException
+     */
+    public function moreLikeThis(array $like, array $fields = [], $options = [])
+    {
+        $moreLikeThis = $options;
+        $moreLikeThis['like'] = $like;
+
+        if (!empty($fields)) {
+            $moreLikeThis['fields'] = $fields;
+        }
+        
+        $this->appendAtPath('query.bool.filter.bool.must', ['more_like_this' => $moreLikeThis]);
+    }
+
     /**
      * This is an low level method for internal usage.
      *
@@ -119,7 +135,7 @@ public function aggregation($name, array $aggregationDefinition, $parentPath = '
      * @param string $parentPath The parent path to add the sub aggregation to
      * @param string $name The name to identify the resulting aggregation
      * @param array $aggregationConfiguration
-     * @return QueryInterface
+     * @return void
      * @throws Exception\QueryBuildingException
      */
     protected function addSubAggregation($parentPath, $name, $aggregationConfiguration)