feat: record competency status in the same save as a subsection grade

lms/djangoapps/grades/models.py

@@ -477,7 +477,6 @@ def update_or_create_grade(cls, **params):
             grade.first_attempted = first_attempted
             grade.save()
 
-        cls._emit_grade_calculated_event(grade)
         return grade
 
     @classmethod
@@ -498,8 +497,6 @@ def bulk_create_grades(cls, grade_params_iter, user_id, course_key):
 
         grades = [PersistentSubsectionGrade(**params) for params in grade_params_iter]
         grades = cls.objects.bulk_create(grades)
-        for grade in grades:
-            cls._emit_grade_calculated_event(grade)
         return grades
 
     @classmethod
@@ -525,10 +522,6 @@ def _prepare_params_visible_blocks_id(cls, params):
         params['visible_blocks_id'] = params['visible_blocks'].hash_value
         del params['visible_blocks']
 
-    @staticmethod
-    def _emit_grade_calculated_event(grade):
-        events.subsection_grade_calculated(grade)
-
     @classmethod
     def _cache_key(cls, course_id):
         return f"subsection_grades_cache.{course_id}"

lms/djangoapps/grades/rest_api/v1/tests/test_gradebook_views.py

@@ -7,6 +7,7 @@
 from collections import OrderedDict, namedtuple
 from contextlib import contextmanager
 from datetime import datetime
+from decimal import Decimal
 from unittest.mock import MagicMock, patch
 
 import ddt
@@ -17,6 +18,7 @@
 from edx_toggles.toggles.testutils import override_waffle_flag
 from freezegun import freeze_time
 from opaque_keys.edx.locator import BlockUsageLocator
+from openedx_learning.api import GradedObjectScore
 from pytz import UTC
 from rest_framework import status
 from rest_framework.response import Response
@@ -1721,6 +1723,29 @@ def test_subsection_does_not_exist(self, login_method):
             assert status.HTTP_422_UNPROCESSABLE_ENTITY == resp.status_code
             assert expected_data == resp.data
 
+    @override_settings(ENABLE_COMPETENCY_MASTERY_TRACKING=True)
+    def test_override_for_unattempted_learner_records_competency_status(self):
+        """
+        A staff-entered grade counts toward competency status even though the learner never attempted the subsection.
+        """
+        subsection = self.subsections[self.chapter_1.location][0]
+        post_data = [{
+            'user_id': self.student.id,
+            'usage_id': str(subsection.location),
+            'grade': {'earned_graded_override': 1, 'possible_graded_override': 2},
+        }]
+
+        with override_waffle_flag(self.waffle_flag, active=True):
+            self.login_staff()
+            with patch('lms.djangoapps.grades.subsection_grade.record_graded_object_statuses') as mock_record:
+                resp = self.client.post(self.get_url(), data=json.dumps(post_data), content_type='application/json')
+
+        assert status.HTTP_202_ACCEPTED == resp.status_code
+        mock_record.assert_any_call(
+            user_id=self.student.id,
+            scores=[GradedObjectScore(object_id=str(subsection.location), fraction=Decimal('0.5'))],
+        )
+
     @ddt.data('login_staff', 'login_course_staff', 'login_course_admin')
     def test_override_is_created(self, login_method):
         """

lms/djangoapps/grades/subsection_grade.py

@@ -6,11 +6,16 @@
 from abc import ABCMeta
 from collections import OrderedDict
 from datetime import datetime, timezone
+from decimal import Decimal
 from logging import getLogger
 
+from django.conf import settings
+from django.db import transaction
 from lazy import lazy
+from openedx_learning.api import GradedObjectScore, record_graded_object_statuses
 from xblock.scorable import ShowCorrectness
 
+from lms.djangoapps.grades import events
 from lms.djangoapps.grades.models import BlockRecord, PersistentSubsectionGrade
 from lms.djangoapps.grades.scores import compute_percent, get_score, possibly_scored
 from xmodule import block_metadata_utils, graders  # pylint: disable=wrong-import-order
@@ -19,6 +24,36 @@
 log = getLogger(__name__)
 
 
+def _enqueue_competency_rollup(user_id, object_ids):
+    """
+    Queues the competency rollup for the scored objects once their grade transaction has committed.
+    """
+    # Imported here because tasks.py imports this module through the subsection grade factory.
+    from lms.djangoapps.grades.tasks import roll_up_competency_statuses_for_user
+
+    try:
+        roll_up_competency_statuses_for_user.apply_async(kwargs={"user_id": user_id, "object_ids": object_ids})
+    except Exception:  # pylint: disable=broad-exception-caught
+        # The grade has committed, so a lost rollup leaves the status stale; support needs the ids to recompute it.
+        log.exception(
+            "Could not queue competency rollup for user %s and objects %s; recompute manually.",
+            user_id, object_ids,
+        )
+
+
+def _record_competency_status(student, grades):
+    """
+    Records competency status inside the caller's transaction and returns the object ids to roll up, or an empty
+    list when tracking is off, nothing is scored, or no rows changed.
+    """
+    if not settings.ENABLE_COMPETENCY_MASTERY_TRACKING:
+        return []
+    scores = [score for score in (grade.graded_object_score() for grade in grades) if score]
+    if not scores or not record_graded_object_statuses(user_id=student.id, scores=scores):
+        return []
+    return [score.object_id for score in scores]
+
+
 class SubsectionGradeBase(metaclass=ABCMeta):  # noqa: B024
     """
     Abstract base class for Subsection Grades.
@@ -285,7 +320,10 @@ def update_or_create_model(self, student, score_deleted=False, force_update_subs
         """
         Saves or updates the subsection grade in a persisted model.
         """
-        if self._should_persist_per_attempted(score_deleted, force_update_subsections):
+        if not self._should_persist_per_attempted(score_deleted, force_update_subsections):
+            return None
+
+        with transaction.atomic():
             model = PersistentSubsectionGrade.update_or_create_grade(**self._persisted_model_params(student))
 
             if hasattr(model, 'override'):
@@ -295,21 +333,57 @@ def update_or_create_model(self, student, score_deleted=False, force_update_subs
                 # the aggregated scores for this object to reflect the override.
                 self.all_total = self._aggregated_score_from_model(model, is_graded=False)
                 self.graded_total = self._aggregated_score_from_model(model, is_graded=True)
+                self.override = model.override
+
+            object_ids = _record_competency_status(student, [self])
+            if object_ids:
+                # robust: a broker failure after the grade committed must not skip the event or fail the save.
+                transaction.on_commit(lambda: _enqueue_competency_rollup(student.id, object_ids), robust=True)
 
-            return model
+        events.subsection_grade_calculated(model)
+        return model
 
     @classmethod
     def bulk_create_models(cls, student, subsection_grades, course_key):
         """
         Saves the subsection grade in a persisted model.
         """
-        params = [
-            subsection_grade._persisted_model_params(student)  # pylint: disable=protected-access
+        persisted_grades = [
+            subsection_grade
             for subsection_grade in subsection_grades
             if subsection_grade
             if subsection_grade._should_persist_per_attempted()  # pylint: disable=protected-access
         ]
-        return PersistentSubsectionGrade.bulk_create_grades(params, student.id, course_key)
+        params = [
+            subsection_grade._persisted_model_params(student)  # pylint: disable=protected-access
+            for subsection_grade in persisted_grades
+        ]
+        if not persisted_grades:
+            return None
+
+        with transaction.atomic():
+            grades = PersistentSubsectionGrade.bulk_create_grades(params, student.id, course_key)
+            object_ids = _record_competency_status(student, persisted_grades)
+            if object_ids:
+                # robust: a broker failure after the grades committed must not skip the events or fail the save.
+                transaction.on_commit(lambda: _enqueue_competency_rollup(student.id, object_ids), robust=True)
+
+        for grade in grades or []:
+            events.subsection_grade_calculated(grade)
+        return grades
+
+    def graded_object_score(self):
+        """
+        Returns the graded fraction as a score, or None when nothing was attempted or nothing could be earned.
+
+        A staff-entered graded override counts as an attempt, as it does in the gradebook.
+        """
+        has_graded_override = self.override is not None and self.override.earned_graded_override is not None
+        if not (self.attempted_graded or has_graded_override) or not self.graded_total.possible:
+            return None
+        # An override can push earned outside [0, possible]; core rejects fractions outside [0, 1].
+        fraction = min(max(Decimal(str(self.percent_graded)), Decimal(0)), Decimal(1))
+        return GradedObjectScore(object_id=str(self.location), fraction=fraction)
 
     def _should_persist_per_attempted(self, score_deleted=False, force_update_subsections=False):
         """

lms/djangoapps/grades/tasks.py

@@ -15,6 +15,7 @@
 )
 from opaque_keys.edx.keys import CourseKey, UsageKey
 from opaque_keys.edx.locator import CourseLocator
+from openedx_learning.api import roll_up_competency_statuses
 from submissions import api as sub_api
 
 from common.djangoapps.student.models import CourseEnrollment
@@ -185,6 +186,29 @@ def recalculate_subsection_grade_v3(self, **kwargs):
     _recalculate_subsection_grade(self, **kwargs)
 
 
+@shared_task(
+    bind=True,
+    base=LoggedPersistOnFailureTask,
+    time_limit=SUBSECTION_GRADE_TIMEOUT_SECONDS,
+    max_retries=2,
+    default_retry_delay=RETRY_DELAY_SECONDS
+)
+def roll_up_competency_statuses_for_user(self, **kwargs):
+    """
+    Rolls up a learner's competency statuses for the graded objects whose criteria were just updated.
+
+    Keyword Arguments:
+        user_id (int): id of applicable User object
+        object_ids (list of string): usage keys of the graded objects that changed
+    """
+    try:
+        roll_up_competency_statuses(user_id=kwargs['user_id'], object_ids=kwargs['object_ids'])
+    except Exception as exc:
+        if not isinstance(exc, KNOWN_RETRY_ERRORS):
+            raise
+        raise self.retry(kwargs=kwargs, exc=exc) from exc
+
+
 def _recalculate_subsection_grade(self, **kwargs):
     """
     Updates a saved subsection grade.

lms/djangoapps/grades/tests/test_api.py

@@ -1,17 +1,22 @@
 """ Tests calling the grades api directly """
 
 
+from decimal import Decimal
 from unittest.mock import patch
 
 import ddt
+from django.test import override_settings
+from openedx_learning.api import GradedObjectScore
 
+from common.djangoapps.student.models import CourseEnrollment
 from common.djangoapps.student.tests.factories import UserFactory
 from lms.djangoapps.grades import api
 from lms.djangoapps.grades.models import (
     PersistentCourseGrade,
     PersistentSubsectionGrade,
     PersistentSubsectionGradeOverride,
 )
+from lms.djangoapps.grades.tests.utils import mock_get_score
 from xmodule.modulestore.tests.django_utils import (
     ModuleStoreTestCase,  # pylint: disable=wrong-import-order
 )
@@ -67,6 +72,23 @@ def tearDown(self):
         super().tearDown()
         PersistentSubsectionGradeOverride.objects.all().delete()
 
+    @override_settings(ENABLE_COMPETENCY_MASTERY_TRACKING=True)
+    def test_override_without_existing_grade_records_competency_status(self):
+        CourseEnrollment.enroll(self.user, self.course.id)
+        new_subsection = BlockFactory.create(parent=self.course, category="sequential", display_name="New Subsection")
+        BlockFactory.create(parent=new_subsection, category="problem", display_name="New Problem")
+        score = GradedObjectScore(object_id=str(new_subsection.location), fraction=Decimal('0.5'))
+
+        with mock_get_score(1, 2), patch(
+            'lms.djangoapps.grades.subsection_grade.record_graded_object_statuses'
+        ) as mock_record:
+            api.override_subsection_grade(
+                self.user.id, self.course.id, new_subsection.location, overrider=self.overriding_user,
+                earned_graded=1.0,
+            )
+
+        mock_record.assert_called_once_with(user_id=self.user.id, scores=[score])
+
     @ddt.data(0.0, None, 3.0)
     def test_override_subsection_grade(self, earned_graded):
         api.override_subsection_grade(

lms/djangoapps/grades/tests/test_course_grade_factory.py

@@ -73,7 +73,7 @@ def _assert_section_order(course_grade):
         with self.assertNumQueries(3), mock_get_score(1, 2):
             _assert_read(expected_pass=False, expected_percent=0)  # start off with grade of 0
 
-        num_queries = 44
+        num_queries = 48
         with self.assertNumQueries(num_queries), mock_get_score(1, 2):
             grade_factory.update(self.request.user, self.course, force_update_subsections=True)
 
@@ -87,14 +87,14 @@ def _assert_section_order(course_grade):
         with self.assertNumQueries(3):
             _assert_read(expected_pass=True, expected_percent=0.5)  # NOT updated to grade of .25
 
-        num_queries = 20
+        num_queries = 24
         with self.assertNumQueries(num_queries), mock_get_score(2, 2):
             grade_factory.update(self.request.user, self.course, force_update_subsections=True)
 
         with self.assertNumQueries(3):
             _assert_read(expected_pass=True, expected_percent=1.0)  # updated to grade of 1.0
 
-        num_queries = 30
+        num_queries = 34
         with self.assertNumQueries(num_queries), mock_get_score(0, 0):  # the subsection now is worth zero
             grade_factory.update(self.request.user, self.course, force_update_subsections=True)
 

lms/djangoapps/grades/tests/test_models.py

@@ -286,16 +286,6 @@ def test_unattempted_save_does_not_remove_attempt(self):
         assert isinstance(grade.first_attempted, datetime)
         assert grade.earned_all == 6.0
 
-    def test_update_or_create_event(self):
-        with patch('lms.djangoapps.grades.events.tracker') as tracker_mock:
-            grade = PersistentSubsectionGrade.update_or_create_grade(**self.params)
-        self._assert_tracker_emitted_event(tracker_mock, grade)
-
-    def test_create_event(self):
-        with patch('lms.djangoapps.grades.events.tracker') as tracker_mock:
-            grade = PersistentSubsectionGrade.update_or_create_grade(**self.params)
-        self._assert_tracker_emitted_event(tracker_mock, grade)
-
     def test_grade_override(self):
         """
         Creating a subsection grade override should NOT change the score values
@@ -322,30 +312,6 @@ def test_grade_override(self):
         assert grade.possible_all == override.possible_all_override
         assert grade.possible_graded == override.possible_graded_override
 
-    def _assert_tracker_emitted_event(self, tracker_mock, grade):
-        """
-        Helper function to ensure that the mocked event tracker
-        was called with the expected info based on the passed grade.
-        """
-        tracker_mock.emit.assert_called_with(
-            'edx.grades.subsection.grade_calculated',
-            {
-                'user_id': str(grade.user_id),
-                'course_id': str(grade.course_id),
-                'block_id': str(grade.usage_key),
-                'course_version': str(grade.course_version),
-                'weighted_total_earned': grade.earned_all,
-                'weighted_total_possible': grade.possible_all,
-                'weighted_graded_earned': grade.earned_graded,
-                'weighted_graded_possible': grade.possible_graded,
-                'first_attempted': str(grade.first_attempted),
-                'subtree_edited_timestamp': str(grade.subtree_edited_timestamp),
-                'event_transaction_id': str(get_event_transaction_id()),
-                'event_transaction_type': str(get_event_transaction_type()),
-                'visible_blocks_hash': str(grade.visible_blocks_id),
-            }
-        )
-
     def test_clear_subsection_grade(self):
         PersistentSubsectionGrade.update_or_create_grade(**self.params)
         deleted = PersistentSubsectionGrade.delete_subsection_grades_for_learner(