feat: remove docker-compose from devstack (#1172)

Author: Rebecca Graber

.github/workflows/cli-tests.yml

@@ -48,15 +48,17 @@ jobs:
           sudo apt update
           sudo apt install docker-ce containerd.io
           docker version
-          docker-compose --version
+          docker compose --version
 
       # Note: we cannot use Docker Desktop because it has not been licensed for use in GithubActions
       - name: Docker installation - Mac
         if: ${{ matrix.os.name == 'mac' }}
         run: |
-          brew install lima docker
+          brew install lima docker docker-compose
           limactl start --name=default template://docker
           echo "DOCKER_HOST=unix:///Users/runner/.lima/default/sock/docker.sock" >> $GITHUB_ENV
+          mkdir -p ~/.docker/cli-plugins
+          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
 
       - name: Install Python dependencies
         run:  make requirements

.github/workflows/provisioning-tests.yml

@@ -28,7 +28,7 @@ jobs:
         os:
           - ubuntu-20.04 # Ubuntu 20.04 "Focal Fossa"
         python-version: [ '3.8' ]
-        services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue]
+        services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms]
       fail-fast: false # some services can be flaky; let others run to completion even if one fails
 
     steps:
@@ -48,7 +48,7 @@ jobs:
           sudo apt update
           sudo apt install docker-ce containerd.io
           docker version
-          docker-compose --version
+          docker compose --version
 
       - name: free up disk space
         run:  sudo apt remove --purge -y ghc-* azure-cli google-cloud-sdk hhvm llvm-* dotnet-* powershell mono-* php* ruby*

Makefile

@@ -189,7 +189,7 @@ dev.pull.without-deps.%: ## Pull latest Docker images for specific services.
 	@scripts/send_metrics.py wrap "dev.pull.without-deps.$*"
 
 impl-dev.pull.without-deps.%: ## Pull latest Docker images for specific services.
-	docker-compose pull $$(echo $* | tr + " ")
+	docker compose pull $$(echo $* | tr + " ")
 
 dev.pull:
 	@scripts/send_metrics.py wrap "$@"
@@ -205,7 +205,7 @@ dev.pull.%: ## Pull latest Docker images for services and their dependencies.
 	@scripts/send_metrics.py wrap "dev.pull.$*"
 
 impl-dev.pull.%: ## Pull latest Docker images for services and their dependencies.
-	docker-compose pull --include-deps $$(echo $* | tr + " ")
+	docker compose pull --include-deps $$(echo $* | tr + " ")
 
 ########################################################################################
 # Developer interface: Database management.
@@ -253,18 +253,18 @@ $(foreach db_service,$(DB_SERVICES_LIST),\
 dev.migrate: | $(_db_migration_targets) ## Run migrations for applicable default services.
 
 dev.migrate.cms:
-	docker-compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms'
+	docker compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms'
 
 dev.migrate.lms:
-	docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms'
+	docker compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms'
 
 dev.migrate.%: ## Run migrations on a service.
-	docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate'
+	docker compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate'
 
 dev.drop-db: _expects-database.dev.drop-db
 
 dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container.
-	docker-compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\""
+	docker compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\""
 
 
 ########################################################################################
@@ -274,7 +274,7 @@ dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mys
 dev.up.attach: _expects-service.dev.up.attach
 
 impl-dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it.
-	docker-compose up $*
+	docker compose up $*
 
 dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it.
 	@scripts/send_metrics.py wrap "dev.up.attach.$*"
@@ -298,7 +298,7 @@ dev.up.with-watchers.%: ## Bring up services and their dependencies + asset watc
 dev.up.without-deps: _expects-service-list.dev.up.without-deps
 
 impl-dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves.
-	docker-compose up -d --no-deps $$(echo $* | tr + " ")
+	docker compose up -d --no-deps $$(echo $* | tr + " ")
 
 dev.up.without-deps.%: ## Bring up services by themselves.
 	@scripts/send_metrics.py wrap "dev.up.without-deps.$*"
@@ -316,7 +316,7 @@ dev.up.large-and-slow: dev.up.$(DEFAULT_SERVICES) ## Bring up default services.
 	@echo # at least one statement so that dev.up.% doesn't run too
 
 impl-dev.up.%: dev.check-memory ## Bring up services and their dependencies.
-	docker-compose up -d $$(echo $* | tr + " ")
+	docker compose up -d $$(echo $* | tr + " ")
 ifeq ($(ALWAYS_CACHE_PROGRAMS),true)
 	make dev.cache-programs
 endif
@@ -326,32 +326,32 @@ dev.up.%:
 	@scripts/send_metrics.py wrap "dev.up.$*"
 
 dev.ps: ## View list of created services and their statuses.
-	docker-compose ps
+	docker compose ps
 
 dev.print-container.%: ## Get the ID of the running container for a given service.
 	@# Can be run as ``make --silent --no-print-directory dev.print-container.$service`` for just ID.
-	@echo $$(docker-compose ps --quiet $*)
+	@echo $$(docker compose ps --quiet $*)
 
 dev.restart-container: ## Restart all service containers.
-	docker-compose restart $$(echo $* | tr + " ")
+	docker compose restart $$(echo $* | tr + " ")
 
 dev.restart-container.%: ## Restart specific services' containers.
-	docker-compose restart $$(echo $* | tr + " ")
+	docker compose restart $$(echo $* | tr + " ")
 
 dev.stop: ## Stop all running services.
-	docker-compose stop
+	docker compose stop
 
 dev.stop.%: ## Stop specific services.
-	docker-compose stop $$(echo $* | tr + " ")
+	docker compose stop $$(echo $* | tr + " ")
 
 dev.kill: ## Kill all running services.
-	docker-compose stop
+	docker compose stop
 
 dev.kill.%: ## Kill specific services.
-	docker-compose kill $$(echo $* | tr + " ")
+	docker compose kill $$(echo $* | tr + " ")
 
 dev.rm-stopped: ## Remove stopped containers. Does not affect running containers.
-	docker-compose rm --force
+	docker compose rm --force
 
 dev.down: ## Documentation for a change to naming
 	@echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details."
@@ -360,10 +360,10 @@ dev.down.%:
 	@echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details."
 
 dev.remove-containers: ## Stop and remove containers and networks for all services.
-	docker-compose down
+	docker compose down
 
 dev.remove-containers.%: ## Stop and remove containers for specific services.
-	docker-compose rm --force --stop $$(echo $* | tr + " ")
+	docker compose rm --force --stop $$(echo $* | tr + " ")
 
 
 ########################################################################################
@@ -382,7 +382,7 @@ dev.check.%:  # Run checks for a given service or set of services.
 	$(WINPTY) bash ./check.sh $*
 
 dev.validate: ## Print effective Docker Compose config, validating files in COMPOSE_FILE.
-	docker-compose config
+	docker compose config
 
 
 ########################################################################################
@@ -395,20 +395,20 @@ dev.cache-programs: ## Copy programs from Discovery to Memcached for use in LMS.
 dev.restart-devserver: _expects-service.dev.restart-devserver
 
 dev.restart-devserver.forum:
-	docker-compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")'
+	docker compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")'
 
 dev.forum.build-indices: ## Build indices for forum service
-	docker-compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices"
+	docker compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices"
 
 dev.restart-devserver.%: ## Kill an edX service's development server. Watcher should restart it.
 	# Applicable to Django services only.
-	docker-compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")'
+	docker compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")'
 
 dev.logs: ## View logs from running containers.
-	docker-compose logs -f
+	docker compose logs -f
 
 dev.logs.%: ## View the logs of the specified service container.
-	docker-compose logs -f --tail=500 $*
+	docker compose logs -f --tail=500 $*
 
 dev.attach: _expects-service.dev.attach
 
@@ -418,54 +418,54 @@ dev.attach.%: ## Attach to the specified service container process for debugging
 dev.shell: _expects-service.dev.shell
 
 dev.shell.credentials:
-	docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'
+	docker compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'
 
 dev.shell.discovery:
-	docker-compose exec discovery env TERM=$(TERM) bash -c '/bin/bash'
+	docker compose exec discovery env TERM=$(TERM) bash -c '/bin/bash'
 
 dev.shell.ecommerce:
-	docker-compose exec ecommerce env TERM=$(TERM) /bin/bash
+	docker compose exec ecommerce env TERM=$(TERM) /bin/bash
 
 dev.shell.registrar:
-	docker-compose exec registrar env TERM=$(TERM) /bin/bash
+	docker compose exec registrar env TERM=$(TERM) /bin/bash
 
 dev.shell.xqueue:
-	docker-compose exec xqueue env TERM=$(TERM) /bin/bash
+	docker compose exec xqueue env TERM=$(TERM) /bin/bash
 
 dev.shell.lms:
-	docker-compose exec lms env TERM=$(TERM) bash -c '/bin/bash'
+	docker compose exec lms env TERM=$(TERM) bash -c '/bin/bash'
 
 dev.shell.lms_watcher:
-	docker-compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash'
+	docker compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash'
 
 dev.shell.cms:
-	docker-compose exec cms env TERM=$(TERM) bash -c '/bin/bash'
+	docker compose exec cms env TERM=$(TERM) bash -c '/bin/bash'
 
 dev.shell.cms_watcher:
-	docker-compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash'
+	docker compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash'
 
 dev.shell.xqueue_consumer:
-	docker-compose exec xqueue_consumer env TERM=$(TERM) /bin/bash
+	docker compose exec xqueue_consumer env TERM=$(TERM) /bin/bash
 
 dev.shell.analyticsapi:
 	docker exec -it edx.devstack.analyticsapi env TERM=$(TERM) bash -c '/bin/bash'
 
 dev.shell.insights:
-	docker-compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash'
+	docker compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash'
 
 dev.shell.%: ## Run a shell on the specified service's container.
-	docker-compose exec $* /bin/bash
+	docker compose exec $* /bin/bash
 
 dev.dbshell:
-	docker-compose exec mysql57 bash -c "mysql"
+	docker compose exec mysql57 bash -c "mysql"
 
 dev.dbcopy8.%: ## Copy data from old mysql 5.7 container into a new 8 db
-	docker-compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql
-	docker-compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql
+	docker compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql
+	docker compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql
 	rm .dev/$*.sql
 
 dev.dbshell.%: ## Run a SQL shell on the given database.
-	docker-compose exec mysql57 bash -c "mysql $*"
+	docker compose exec mysql57 bash -c "mysql $*"
 
 # List of Makefile targets to run static asset generation, in the form dev.static.$(service)
 # Services will only have their asset generation added here
@@ -478,13 +478,13 @@ $(foreach asset_service,$(ASSET_SERVICES_LIST),\
 dev.static: | $(_asset_compilation_targets)
 
 dev.static.lms:
-	docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms'
+	docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms'
 
 dev.static.cms:
-	docker-compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms'
+	docker compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms'
 
 dev.static.%: ## Rebuild static assets for the specified service's container.
-	docker-compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'
+	docker compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'
 
 
 ########################################################################################
@@ -605,18 +605,18 @@ docs: ## generate Sphinx HTML documentation, including API docs
 
 validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container.
 	touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile
-	docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile
+	docker compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile
 	rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile
 
 vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers.
-	@docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1
-	@docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1
+	@docker compose logs chrome 2>&1 | grep "VNC password" | tail -1
+	@docker compose logs firefox 2>&1 | grep "VNC password" | tail -1
 
 devpi-password: ## Get the root devpi password for the devpi container.
-	docker-compose exec devpi bash -c "cat /data/server/.serverpassword"
+	docker compose exec devpi bash -c "cat /data/server/.serverpassword"
 
 hadoop-application-logs-%: ## View hadoop logs by application Id.
-	docker-compose exec nodemanager yarn logs -applicationId $*
+	docker compose exec nodemanager yarn logs -applicationId $*
 
 create-test-course: ## Provisions cms, and ecommerce with course(s) in test-course.json.
 	$(WINPTY) bash ./course-generator/create-courses.sh --cms --ecommerce course-generator/test-course.json

check.sh

@@ -46,7 +46,7 @@ run_check() {
     if $cmd; then  # Run the command itself and check if it succeeded.
         succeeded="$succeeded $check_name"
     else
-        docker-compose logs "$service"
+        docker compose logs "$service"
         failed="$failed $check_name"
     fi
     set -e  # Re-enable exit-on-error

course-generator/create-courses.sh

@@ -7,14 +7,14 @@ echo "Parsing options"
 container_error=false
 for arg in "$@"; do
     if [ $arg == "--cms" ]; then
-        if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then
+        if [ ! "$(docker compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then
             echo "Issue with cms container"
             container_error=true
         else
             cms=true
         fi
     elif [ $arg == "--ecommerce" ]; then
-        if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
+        if [ ! "$(docker compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
             echo "Issue with ecommerce container"
             container_error=true
         else
@@ -42,10 +42,10 @@ done < "${@: -1}"
 
 if $cms ; then
 	echo "Creating courses on cms."
-	docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'"
+	docker compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'"
 fi
 
 if $ecommerce ; then
 	echo "Creating courses on ecommerce."
-	docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
+	docker compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
 fi