feat: add optional coursegraph service (#820)

Add a "coursegraph" service to devstack using
the official Neo4j Docker image.

The service is "extra"; that is, it will not start, be
pulled, or be provisioned by default. Instead,
provisioning can be done on-demand,
which will dump data from LMS to Coursegraph:

  make dev.provision.coursegraph

Commands like `dev.up.<service>`, `dev.attach.<service>`, etc
are available for coursegraph, as they are for any other
Devstack service. Further documentation will live in the
edx-platform repo, in:

  ./openedx/core/djangoapps/coursegraph/README.rst

Why add Coursegraph? It is a tool that is relied upon
by many edX developers and support specialists,
yet very few of us know how to operate it or fix it
when it goes down. By making it part of devstack,
it will be easier to debug issues and test out Neo4j
upgrades. Furthermore, this enables developers
to trial complex Cypher queries on custom data
instead of simply trusting what they see returned
by production Coursegraph.

TNL-8386

Author: kdmccormick

Makefile

@@ -255,21 +255,23 @@ impl-dev.provision.%: dev.check-memory ## Provision specified services.
 dev.provision.%: ## Provision specified services.
 	@scripts/send_metrics.py wrap "dev.provision.$*"
 
-dev.backup: dev.up.mysql+mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710 ## Write all data volumes to the host.
+dev.backup: dev.up.mysql+mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+coursegraph ## Write all data volumes to the host.
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql57.tar.gz /var/lib/mysql
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch7.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch710) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch710.tar.gz /usr/share/elasticsearch/data
+	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.coursegraph) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/coursegraph.tar.gz /data
 
-dev.restore: dev.up.mysql+mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710 ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA!
+dev.restore: dev.up.mysql+mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+coursegraph ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA!
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql57.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch7.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch710) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch710.tar.gz
+	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.coursegraph) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/coursegraph.tar.gz
 
 # List of Makefile targets to run database migrations, in the form dev.migrate.$(service)
 # Services will only have their migrations added here
@@ -515,6 +517,9 @@ dev.static.%: ## Rebuild static assets for the specified service's container.
 
 dev.reset: dev.down dev.reset-repos dev.prune dev.pull.large-and-slow dev.up.large-and-slow dev.static dev.migrate ## Attempt to reset the local devstack to the master working state without destroying data.
 
+dev.destroy.coursegraph: dev.down.coursegraph ## Remove all coursegraph data.
+	docker volume rm ${COMPOSE_PROJECT_NAME}_coursegraph_data
+
 dev.destroy: ## Irreversibly remove all devstack-related containers, networks, and volumes.
 	$(WINPTY) bash ./destroy.sh
 

README.rst

@@ -312,6 +312,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro
 +------------------------------------+-------------------------------------+----------------+--------------+
 | `xqueue`_                          | http://localhost:18040/api/v1/      | Python/Django  | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
+| `coursegraph`                      | http://localhost:7474/browser       | Tooling (Java) | Extra        |
++------------------------------------+-------------------------------------+----------------+--------------+
 
 Some common service combinations include:
 
@@ -336,6 +338,7 @@ Some common service combinations include:
 .. _frontend-app-library-authoring: https://github.com/edx/frontend-app-library-authoring
 .. _frontend-app-course-authoring: https://github.com/edx/frontend-app-course-authoring
 .. _xqueue: https://github.com/edx/xqueue
+.. _coursegraph: https://github.com/edx/edx-platform/tree/master/openedx/core/djangoapps/coursegraph
 
 
 Known Issues

docker-compose.yml

@@ -31,6 +31,26 @@ services:
       - ../edx-e2e-tests/upload_files:/edx/app/e2e/edx-e2e-tests/upload_files
       - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data
 
+  coursegraph:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.coursegraph"
+    hostname: coursegraph.devstack.edx
+    # Try to keep this in sync with the NEO4J_VERSION declared within
+    # https://github.com/edx/configuration/blob/master/playbooks/roles/neo4j
+    image: neo4j:3.2
+    networks:
+      default:
+        aliases:
+          - edx.devstack.coursegraph
+    ports:
+      - "7474:7474"  # Expose Web interface at http://localhost:7474.
+      - "7687:7687"  # Expose Bolt interface at bolt://user:password@localhost:7687.
+    volumes:
+      - coursegraph_data:/data
+    stdin_open: true
+    tty: true
+    environment:
+      NEO4J_AUTH: "neo4j/edx"  # Initial username/password for Neo4j Web interface.
+
   devpi:
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi"
     hostname: devpi.devstack.edx
@@ -600,6 +620,7 @@ services:
       - lms
 
 volumes:
+  coursegraph_data:
   discovery_assets:
   devpi_data:
   edxapp_lms_assets:

options.mk

@@ -97,4 +97,4 @@ credentials+discovery+ecommerce+lms+registrar+studio
 # All third-party services.
 # Separated by plus signs. Listed in alphabetical order for clarity.
 THIRD_PARTY_SERVICES ?= \
-chrome+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql+mysql57+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica
+chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql+mysql57+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica

provision-coursegraph.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+. scripts/colors.sh
+set -x
+
+echo -e "${GREEN}   Ensuring Coursegraph image is up to date...${NC}"
+
+# Pulling the image will almost always be a no-op, but will be important
+# when we bump the version in docker-compose.yml or when Neo4j releases a patch.
+# Also, this gives us a chance to refresh the container in case it's gotten into
+# a weird state.
+docker-compose rm --force --stop coursegraph
+docker-compose pull coursegraph
+docker-compose up -d coursegraph
+
+sleep 10  # Give Neo4j some time to boot up.
+
+echo -e "${GREEN}   Updating LMS courses in Coursegraph...${NC}"
+
+docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && ./manage.py lms dump_to_neo4j --host coursegraph.devstack.edx --user neo4j --password edx'
+
+echo -e "${GREEN}   Coursegraph is now up-to-date with LMS!${NC}"

provision.sh

@@ -47,6 +47,7 @@ forum \
 notes \
 registrar \
 xqueue \
+coursegraph \
  "
 
 # What should we provision?