Gh-322: Added gremlin gaffer image and compose (#323)

* Added gremlin gaffer image and compose
---------

Co-authored-by: GCHQDeveloper314 <[email protected]>

docker/gremlin-gaffer/.env

@@ -0,0 +1,7 @@
+ZOOKEEPER_VERSION=3.7.1
+GAFFER_VERSION=2.0.0
+GREMLIN_VERSION=3.6.4
+ACCUMULO_VERSION=2.0.1
+HADOOP_VERSION=3.3.3
+ACCUMULO_CONF_DIR=/etc/accumulo/conf
+HADOOP_CONF_DIR=/etc/hadoop/conf

docker/gremlin-gaffer/Dockerfile

@@ -0,0 +1,51 @@
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BUILDER_IMAGE_NAME=maven
+ARG BUILDER_IMAGE_TAG=3.8.4-jdk-8
+
+ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server
+ARG BASE_IMAGE_TAG=3.6.4
+
+FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
+
+ARG BASE_IMAGE_TAG
+ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
+ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
+
+WORKDIR /jars
+
+# Allow users to provide their own JAR files
+COPY ./files/ .
+# Try to download required version from Maven Central, otherwise build from source
+RUN	allFilesDownloaded="TRUE" && \
+	if [ ! -f "./tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ] && [ "${allFilesDownloaded}" = "TRUE" ]; then \
+		wget -nv "${GAFFER_DOWNLOAD_URL}/uk/gov/gchq/gaffer/tinkerpop/${GAFFER_VERSION}/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" || allFilesDownloaded="FALSE"; \
+	fi && \
+	if [ "${allFilesDownloaded}" = "FALSE" ]; then \
+		git clone ${GAFFER_GIT_REPO} /tmp/gaffer && \
+		cd /tmp/gaffer && \
+		git checkout ${GAFFER_VERSION} || git checkout gaffer2-${GAFFER_VERSION} && \
+		mvn clean package -Pquick -pl :tinkerpop && \
+		if [ ! -f "/jars/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ]; then \
+			cp ./library/tinkerpop/target/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar /jars; \
+		fi \
+	fi
+
+FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
+COPY --from=builder --chown=root:root /jars/*.jar ext/gafferpop/plugin/
+COPY ./conf/gafferpop/ conf/gafferpop/
+COPY ./conf/gremlin-server-empty-gaffer.yaml conf/gremlin-server-empty-gaffer.yaml
+CMD ["conf/gremlin-server-empty-gaffer.yaml"]

docker/gremlin-gaffer/README.md

@@ -0,0 +1,39 @@
+Gremlin Gaffer Plugin
+======
+In this folder you can find the required files for building and running a gremlin-server with the Gaffer plugin loaded.
+
+The Docker image uses TinkerPop's gremlin-server with GafferPop config and plugin jars added in.
+When run with docker compose it will provide you a full accumulo ecosystem complete with [hdfs](../hdfs) and the [Gaffer REST API](../gaffer-rest).
+
+# Running Locally
+The easiest way to build and run these services is to use docker compose, by running the following from this directory:
+```bash
+docker compose up
+```
+
+## Example Notebook
+See `gremlin-gaffer-modern-example.ipynb` for an example using the "TinkerPop Modern" demo graph.
+
+## Customising the build
+Custom Gaffer TinkerPop plugin jars can be added in the files directory. The Gaffer schema, store properties and gafferpop properties can be found in `conf/gafferpop` and are customised in a docker compose build using volumes. The `gremlin-server-empty-gaffer.yaml` cannot be overwritten in a volume, it must be built into the image.
+
+## Containers that are started:
+* Zookeeper
+* HDFS
+    * Datanode
+    * Namenode
+* Accumulo
+    * Monitor
+    * GC
+    * tserver
+    * Master
+* Gaffer REST
+* Gremlin Server with GafferPop
+
+Access the HDFS NameNode web UI at: http://localhost:9870
+
+Access the Accumulo Monitor UI at: http://localhost:9995
+
+Access the Gaffer REST API at: http://localhost:8080/rest/
+
+Access the Gremlin Server with GafferPop at: http://localhost:8182/

docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/elements.json

@@ -0,0 +1,52 @@
+{
+  "entities": {
+    "person": {
+      "vertex": "person",
+      "properties": {
+        "name": "name.string",
+        "age": "age.integer"
+      },
+      "aggregate": false
+    },
+    "software": {
+      "vertex": "software",
+      "properties": {
+        "name": "name.string",
+        "lang": "name.string"
+      },
+      "aggregate": false
+    }
+  },
+  "edges": {
+    "knows": {
+      "source": "person",
+      "destination": "person",
+      "directed": "true",
+      "properties": {
+        "weight": "weight.double"
+      },
+      "aggregate": false
+    },
+    "created": {
+      "source": "person",
+      "destination": "software",
+      "directed": "true",
+      "properties": {
+        "weight": "weight.double"
+      },
+      "aggregate": false
+    },
+    "dependsOn": {
+      "source": "software",
+      "destination": "software",
+      "directed": "true",
+      "aggregate": false
+    },
+    "encapsulates": {
+      "source": "software",
+      "destination": "software",
+      "directed": "true",
+      "aggregate": false
+    }
+  }
+}

docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/types.json

@@ -0,0 +1,22 @@
+{
+  "types": {
+    "person": {
+      "class": "java.lang.String"
+    },
+    "software": {
+      "class": "java.lang.String"
+    },
+    "true": {
+      "class": "java.lang.Boolean"
+    },
+    "name.string": {
+      "class": "java.lang.String"
+    },
+    "age.integer": {
+      "class": "java.lang.Integer"
+    },
+    "weight.double": {
+      "class": "java.lang.Double"
+    }
+  }
+}

docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties

@@ -0,0 +1,24 @@
+#
+# Copyright 2016-2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gaffer.store.class=uk.gov.gchq.gaffer.accumulostore.AccumuloStore
+gaffer.store.properties.class=uk.gov.gchq.gaffer.accumulostore.AccumuloProperties
+accumulo.instance=accumulo
+accumulo.zookeepers=zookeeper
+accumulo.user=root
+accumulo.password=secret
+# General store config
+gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService
+gaffer.store.job.tracker.enabled=true

docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties

@@ -0,0 +1,20 @@
+#
+# Copyright 2016-2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
+gaffer.graphId=graph1
+gaffer.storeproperties=conf/gafferpop/gaffer/store.properties
+gaffer.schemas=conf/gafferpop/gaffer/schema/
+gaffer.userId=user01

docker/gremlin-gaffer/conf/graphConfig.json

@@ -0,0 +1,3 @@
+{ 
+    "graphId": "graph1"
+}

docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml

@@ -0,0 +1,43 @@
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+host: localhost
+port: 8182
+evaluationTimeout: 30000
+graphs: {
+  graph: conf/gafferpop/gafferpop-tinkerpop-modern.properties}
+scriptEngines: {
+  gremlin-groovy: {
+    plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
+               uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {},
+               org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true},
+               org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},
+               org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}}
+  }
+}
+serializers:
+  - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}                                                                 # application/vnd.graphbinary-v1.0-stringd
+
+metrics: {
+  slf4jReporter: {enabled: true, interval: 180000}}
+strictTransactionManagement: false
+idleConnectionTimeout: 0
+keepAliveInterval: 0
+maxInitialLineLength: 4096
+maxHeaderSize: 8192
+maxChunkSize: 8192
+maxContentLength: 10485760
+maxAccumulationBufferComponents: 1024
+resultIterationBatchSize: 64
+enableAuditLog: true