feat: add insights service (#818)

* feat: add insights service

* fix: insights requirements

* style: more consistent shell invoke

* docs: add insights to readme

Author: zacharis278

Makefile

@@ -476,6 +476,9 @@ dev.shell.studio_watcher:
 dev.shell.xqueue_consumer:
 	docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open
 
+dev.shell.insights:
+	docker-compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /edx/app/insights/devstack.sh open'
+
 dev.shell.%: ## Run a shell on the specified service's container.
 	docker-compose exec $* /bin/bash
 

README.rst

@@ -316,6 +316,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro
 +------------------------------------+-------------------------------------+----------------+--------------+
 | `coursegraph`                      | http://localhost:7474/browser       | Tooling (Java) | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
+| `insights`                         | http://localhost:18110              | Python/Django  | Extra        |
++------------------------------------+-------------------------------------+----------------+--------------+
 
 Some common service combinations include:
 

check.sh

@@ -109,6 +109,12 @@ if should_check xqueue; then
         "curl --fail -L http://localhost:18040/xqueue/status"
 fi
 
+if should_check insights; then
+    echo "Running Analytics Dashboard Devstack tests: "
+    run_check insights_heartbeat insights \
+        "curl --fail -L http://localhost:18110/health/"
+fi
+
 echo "Successful checks:${succeeded:- NONE}"
 echo "Failed checks:${failed:- NONE}"
 if [[ "$succeeded" ]]; then

docker-compose-host.yml

@@ -50,6 +50,10 @@ services:
       - edxapp_tox:/edx/app/edxapp/edx-platform/.tox
       - edxapp_uploads:/edx/var/edxapp/uploads
       - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached
+  insights:
+    volumes:
+      - ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights
+      - insights_node_modules:/edx/app/insights/insights/node_modules
 
   # Note that frontends mount `src` to /edx/app/src instead of /edx/src.
   # See ADR #5 for rationale.
@@ -99,6 +103,7 @@ volumes:
   credentials_node_modules:
   discovery_node_modules:
   ecommerce_node_modules:
+  insights_node_modules:
   edxapp_media:
   edxapp_node_modules:
   edxapp_uploads:

docker-compose.yml

@@ -368,6 +368,35 @@ services:
     volumes:
       - edxapp_lms_assets:/edx/var/edxapp/staticfiles/
 
+  insights:
+    command: bash -c 'source /edx/app/insights/insights_env && while true; do python /edx/app/insights/insights/manage.py runserver 0.0.0.0:18110 --settings analytics_dashboard.settings.devstack; sleep 2; done'
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights"
+    hostname: insights.devstack.edx
+    depends_on:
+      - mysql57
+      - lms
+      - memcached
+    # Allows attachment to the insights service using 'docker attach <containerID>'.
+    stdin_open: true
+    tty: true
+    environment:
+      DB_HOST: edx.devstack.mysql57
+      DB_NAME: dashboard
+      DB_PORT: 3306
+      DB_USER: rozencrantz
+      DB_PASSWORD: secret
+      LMS_HOST: http://localhost:18000
+      DJANGO_SETTINGS_MODULE: analytics_dashboard.settings.devstack
+    image: edxops/insights:${OPENEDX_RELEASE:-latest}
+    networks:
+      default:
+        aliases:
+          - edx.devstack.insights
+    ports:
+      - "18110:18110"
+    volumes:
+      - /edx/var/insights/
+ 
   registrar:
     command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar"

options.mk

@@ -74,7 +74,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-gradebook+front
 # Separated by plus signs.
 # Separated by plus signs. Listed in alphabetical order for clarity.
 EDX_SERVICES ?= \
-credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer
+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer
 
 # Services with database migrations.
 # Should be a subset of $(EDX_SERVICES).
@@ -92,7 +92,7 @@ credentials+discovery+ecommerce+lms+registrar+studio
 # Note: This list should contain _all_ services with static asse to compile ts, even if not
 # configured to run; the list will be filtered later against $(DEFAULT_SERVICES).
 ASSET_SERVICES ?= \
-credentials+discovery+ecommerce+lms+registrar+studio
+credentials+discovery+ecommerce+insights+lms+registrar+studio
 
 # All third-party services.
 # Separated by plus signs. Listed in alphabetical order for clarity.

provision-insights.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+. scripts/colors.sh
+set -x
+
+name=insights
+port=8011
+
+docker-compose up -d insights
+
+echo -e "${GREEN}Installing requirements for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make develop' -- ${name}
+
+# # Install Insights npm dependencies
+docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights/ && npm install && ./npm-post-install.sh'
+
+echo -e "${GREEN}Running migrations for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/insights/insights_env && export DJANGO_SETTINGS_MODULE="analytics_dashboard.settings.devstack" && cd /edx/app/insights/insights && make migrate' -- ${name}
+
+./provision-ida-user.sh ${name} ${name} ${port}
+
+# Compile static assets last since they are absolutely necessary for all services. This will allow developers to get
+# started if they do not care about static assets
+echo -e "${GREEN}Compiling static assets for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make static' -- "$name"

provision.sh

@@ -48,6 +48,7 @@ notes \
 registrar \
 xqueue \
 coursegraph \
+insights \
  "
 
 # What should we provision?

provision.sql

@@ -21,4 +21,7 @@ CREATE DATABASE IF NOT EXISTS edxapp_csmh;
 GRANT ALL ON edxapp.* TO 'edxapp001'@'%' IDENTIFIED BY 'password';
 GRANT ALL ON edxapp_csmh.* TO 'edxapp001'@'%';
 
+CREATE DATABASE IF NOT EXISTS `dashboard`;
+GRANT ALL PRIVILEGES ON `dashboard`.* TO 'rozencrantz'@'%' IDENTIFIED BY 'secret';
+
 FLUSH PRIVILEGES;

repo.sh

@@ -29,6 +29,7 @@ repos=(
     "https://github.com/edx/edx-notes-api.git"
     "https://github.com/edx/edx-platform.git"
     "https://github.com/edx/xqueue.git"
+    "https://github.com/edx/edx-analytics-dashboard.git"
     "https://github.com/edx/frontend-app-gradebook.git"
     "https://github.com/edx/frontend-app-payment.git"
     "https://github.com/edx/frontend-app-publisher.git"
@@ -52,6 +53,7 @@ ssh_repos=(
     "git@github.com:edx/edx-notes-api.git"
     "git@github.com:edx/edx-platform.git"
     "git@github.com:edx/xqueue.git"
+    "git@github.com:edx/edx-analytics-dashboard.git"
     "git@github.com:edx/frontend-app-gradebook.git"
     "git@github.com:edx/frontend-app-payment.git"
     "git@github.com:edx/frontend-app-publisher.git"