feat: schedule triggering of provisioning tests and alert OpsGenie on failure (#843)

* feat: schedule triggering of provisioning tests and alert OpsGenie on failure

Author: Rebecca Graber

.github/workflows/provisioning-tests.yml

@@ -8,6 +8,9 @@ on:
   pull_request:
     branches:
       - '**'
+  schedule:
+    # run at 7:30 am M-F
+    - cron: '30 11 * * 1-5'
 
 jobs:
 
@@ -66,5 +69,31 @@ jobs:
       - name: dev.check
         run:  make dev.check.${{matrix.services}}
 
+      - name: notify on failure
+        if: ${{ failure() && github.ref == 'refs/heads/master' }}
+        uses: dawidd6/action-send-mail@v3
+        with:
+           server_address: email-smtp.us-east-1.amazonaws.com
+           server_port: 465
+           username: ${{secrets.EDX_SMTP_USERNAME}}
+           password: ${{secrets.EDX_SMTP_PASSWORD}}
+           subject: 'Failure: Devstack provisioning tests for ${{matrix.services}} #${{github.run_id}}'
+           to: devstack-provisioning-tests@edx.opsgenie.net
+           from: github-actions <github-actions@edx.org>
+           body: 'Devstack provisioning tests in ${{github.repository}} for ${{matrix.services}} failed! For details see "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}". \n Runbook url: https://openedx.atlassian.net/l/c/zoEcLk2z .'
+
+      - name: close alerts on success
+        if: ${{ !failure() && github.ref == 'refs/heads/master' }}
+        uses: dawidd6/action-send-mail@v3
+        with:
+           server_address: email-smtp.us-east-1.amazonaws.com
+           server_port: 465
+           username: ${{secrets.EDX_SMTP_USERNAME}}
+           password: ${{secrets.EDX_SMTP_PASSWORD}}
+           subject: 'Back to normal: Devstack provisioning tests for ${{matrix.services}} #${{github.run_id}}'
+           to: devstack-provisioning-tests@edx.opsgenie.net
+           from: github-actions <github-actions@edx.org>
+           body: Devstack provisioning tests in ${{github.repository}} are back to normal for ${{matrix.services}}
+
       - name: docs
         run:  make docs