Merge pull request #1189 from openedx/zshkoor/mysql8

feat: upgrade mysql to 8.0

Author: mumarkhan999

Makefile

@@ -8,8 +8,8 @@
 #     make dev.attach.credentials
 #     make dev.pull.registrar+cms
 #     make dev.up.lms
-#     make dev.up.without-deps.lms+forum+discovery+mysql57+elasticsearch710+memcached
-#     make dev.restart-container.mysql57+lms
+#     make dev.up.without-deps.lms+forum+discovery+mysql80+elasticsearch710+memcached
+#     make dev.restart-container.mysql80+lms
 
 # There are also "prefix-form" targets, which are simply an alternate way to spell
 # the 'dev.' targets.
@@ -264,7 +264,7 @@ dev.migrate.%: ## Run migrations on a service.
 dev.drop-db: _expects-database.dev.drop-db
 
 dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container.
-	docker compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\""
+	docker compose exec -T mysql80 bash -c "mysql --execute=\"DROP DATABASE $*;\""
 
 
 ########################################################################################
@@ -457,15 +457,19 @@ dev.shell.%: ## Run a shell on the specified service's container.
 	docker compose exec $* /bin/bash
 
 dev.dbshell:
-	docker compose exec mysql57 bash -c "mysql"
+	docker compose exec mysql80 bash -c "mysql"
+
+DB_NAMES_LIST = credentials discovery ecommerce notes registrar xqueue edxapp edxapp_csmh dashboard analytics-api reports reports_v1
+_db_copy8_targets = $(addprefix dev.dbcopy8.,$(DB_NAMES_LIST))
+dev.dbcopyall8: | $(_db_copy8_targets) ## Copy data from old mysql 5.7 containers into new mysql8 dbs
 
 dev.dbcopy8.%: ## Copy data from old mysql 5.7 container into a new 8 db
 	docker compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql
 	docker compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql
 	rm .dev/$*.sql
 
 dev.dbshell.%: ## Run a SQL shell on the given database.
-	docker compose exec mysql57 bash -c "mysql $*"
+	docker compose exec mysql80 bash -c "mysql $*"
 
 # List of Makefile targets to run static asset generation, in the form dev.static.$(service)
 # Services will only have their asset generation added here

configuration_files/ecommerce.yml

@@ -25,7 +25,7 @@ DATABASES:
         ATOMIC_REQUESTS: true
         CONN_MAX_AGE: 60
         ENGINE: django.db.backends.mysql
-        HOST: edx.devstack.mysql57
+        HOST: edx.devstack.mysql80
         NAME: ecommerce
         OPTIONS:
             connect_timeout: 10

configuration_files/xqueue.yml

@@ -5,7 +5,7 @@ DATABASES:
         ATOMIC_REQUESTS: true
         CONN_MAX_AGE: 0
         ENGINE: django.db.backends.mysql
-        HOST: edx.devstack.mysql57
+        HOST: edx.devstack.mysql80
         NAME: xqueue
         OPTIONS: {}
         PASSWORD: password

docker-compose.yml

@@ -293,13 +293,14 @@ services:
     depends_on:
       - lms
       - memcached
+      - mysql80
       - mysql57
     # Allows attachment to the credentials service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
       CACHE_LOCATION: edx.devstack.memcached:11211
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2
       ENABLE_DJANGO_TOOLBAR: 1
       DJANGO_WATCHMAN_TIMEOUT: 30
@@ -318,6 +319,7 @@ services:
     depends_on:
       - elasticsearch710
       - memcached
+      - mysql80
       - mysql57
       - opensearch12
       - redis
@@ -327,7 +329,7 @@ services:
     environment:
       # This next DB_MIGRATION_HOST line can be removed once edx/configuration has been updated with this value for
       # a while and most people have had a chance to do a "make pull" to get the latest images.
-      DB_MIGRATION_HOST: edx.devstack.mysql57
+      DB_MIGRATION_HOST: edx.devstack.mysql80
       TEST_ELASTICSEARCH_URL: "edx.devstack.elasticsearch710"
       ENABLE_DJANGO_TOOLBAR: 1
       DJANGO_WATCHMAN_TIMEOUT: 30
@@ -350,6 +352,7 @@ services:
       - discovery
       - lms
       - memcached
+      - mysql80
       - mysql57
     # Allows attachment to the ecommerce service using 'docker attach <containerID>'.
     stdin_open: true
@@ -377,6 +380,7 @@ services:
       - devpi
       - elasticsearch710
       - lms
+      - mysql80
       - mysql57
     image: openedx/edx-notes-api-dev:${OPENEDX_RELEASE:-latest}
     networks:
@@ -387,7 +391,7 @@ services:
       - "18120:18120"
     environment:
       DB_ENGINE: "django.db.backends.mysql"
-      DB_HOST: "edx.devstack.mysql57"
+      DB_HOST: "edx.devstack.mysql80"
       DB_NAME: "notes"
       DB_PASSWORD: "password"
       DB_PORT: "3306"
@@ -427,6 +431,7 @@ services:
       - forum
       - memcached
       - mongo
+      - mysql80
       - mysql57
     # Allows attachment to the LMS service using 'docker attach <containerID>'.
     stdin_open: true
@@ -463,14 +468,15 @@ services:
     hostname: insights.devstack.edx
     depends_on:
       - analyticsapi
+      - mysql80
       - mysql57
       - lms
       - memcached
     # Allows attachment to the insights service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: dashboard
       DB_PORT: 3306
       DB_USER: analytics001
@@ -495,13 +501,14 @@ services:
     container_name: edx.devstack.analyticsapi
     hostname: analyticsapi
     depends_on:
+      - mysql80
       - mysql57
       - elasticsearch710
     command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true;  do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done'
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_PORT: 3306
       DB_USER: analytics001
       DB_PASSWORD: password
@@ -520,6 +527,7 @@ services:
     depends_on:
       - discovery
       - lms
+      - mysql80
       - mysql57
       - memcached
       - redis
@@ -528,7 +536,7 @@ services:
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: registrar
       DB_PORT: 3306
       DB_USER: registrar001
@@ -562,12 +570,13 @@ services:
     hostname: registrar-worker.devstack.edx
     depends_on:
       - lms
+      - mysql80
       - mysql57
       - redis
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: registrar
       DB_PORT: 3306
       DB_USER: registrar001
@@ -600,6 +609,7 @@ services:
       - lms
       - memcached
       - mongo
+      - mysql80
       - mysql57
     # Allows attachment to the CMS service using 'docker attach <containerID>'.
     stdin_open: true
@@ -639,6 +649,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue
       - ${PWD}/configuration_files/xqueue.yml:/edx/etc/xqueue.yml
     depends_on:
+      - mysql80
       - mysql57
     environment:
       XQUEUE_CFG: "/edx/etc/xqueue.yml"
@@ -658,6 +669,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue
       - ${PWD}/configuration_files/xqueue.yml:/edx/etc/xqueue.yml
     depends_on:
+      - mysql80
       - mysql57
     networks:
       default:

docs/devstack_faq.rst

@@ -134,7 +134,7 @@ How do I connect to the databases from an outside editor?
 ---------------------------------------------------------
 
 To connect to the databases from an outside editor (such as MySQLWorkbench),
-first uncomment these lines from ``docker-compose.yml``'s ``mysql57`` section
+first uncomment these lines from ``docker-compose.yml``'s ``mysql80`` section
 
 .. code:: yaml
 
@@ -145,8 +145,8 @@ Then bring your mysql container down and back up by running:
 
 .. code:: sh
 
-  docker compose stop mysql57
-  docker compose up -d mysql57
+  docker compose stop mysql80
+  docker compose up -d mysql80
 
 Then connect using the values below. Note that the username and password will
 vary depending on the database. For all of the options, see ``provision.sql``.
@@ -158,7 +158,7 @@ vary depending on the database. For all of the options, see ``provision.sql``.
 
 If you have trouble connecting, ensure the port was mapped successfully by
 running ``make dev.ps`` and looking for a line like this:
-``edx.devstack.mysql57 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``.
+``edx.devstack.mysql80 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``.
 
 How do I build the service images myself?
 -----------------------------------------
@@ -209,7 +209,7 @@ To access the MySQL shell for a particular database, run:
 
 .. code:: sh
 
-   make dev.shell.mysql57
+   make dev.shell.mysql80
    mysql
    use <database>;
 

load-db.sh

@@ -16,6 +16,6 @@ then
 fi
 
 echo "Loading the $1 database..."
-mysql_container=$(make --silent --no-print-directory dev.print-container.mysql57)
+mysql_container=$(make --silent --no-print-directory dev.print-container.mysql80)
 docker exec -i "$mysql_container" mysql -uroot $1 < $1.sql
 echo "Finished loading the $1 database!"

update-dbs-init-sql-scripts.sh

@@ -24,7 +24,7 @@ make dev.pull.lms+ecommerce
 make dev.provision.services.lms+ecommerce
 
 # dump schema and data from mysql databases in the mysql docker container and copy them to current directory in docker host
-MYSQL_DOCKER_CONTAINER="$(make --silent --no-print-directory dev.print-container.mysql57)"
+MYSQL_DOCKER_CONTAINER="$(make --silent --no-print-directory dev.print-container.mysql80)"
 for DB_NAME in "${DBS[@]}"; do
     DB_CREATION_SQL_SCRIPT="${DB_NAME}.sql"
     if [[ " ${EDXAPP_DBS[@]} " =~ " ${DB_NAME} " ]]; then