diff --git a/docs/management.rst b/docs/management.rst new file mode 100644 index 0000000000..19b3e56978 --- /dev/null +++ b/docs/management.rst @@ -0,0 +1,45 @@ +Management Commands +=================== + +The following custom Django management commands are available: + +Regenerate submission JSON +-------------------------- + +Regenerates the JSON for all submissions of a form. + +This is useful in the case where the JSON was saved incorrectly due to some bug when parsing the XML or when saving metadata. + +The form is identified by its ID. + +.. code-block:: bash + + python manage.py regenerate_submission_json form_id1 form_id2 + + +Restore soft deleted form +------------------------- + +Restores a soft deleted form. The form is identified by its ID. + +.. code-block:: bash + + python manage.py restore_form form_id + +You can also restore a form in Django admin interface: + +1. **Navigate to XForms**: Go to the XForm section in the Django admin interface. + +2. **Select Forms**: Select the soft-deleted forms you want to restore. + +3. **Run Action**: Choose the "Restore selected soft-deleted forms" action from the dropdown menu and click "Go". + + +Soft delete user +---------------- + +Softs deletes a user. The user is identified by their username and email + +.. code-block:: bash + + python manage.py delete_users --user_details username1:email username2:email diff --git a/onadata/apps/logger/admin.py b/onadata/apps/logger/admin.py index 76be89c99f..804fdf9a93 100644 --- a/onadata/apps/logger/admin.py +++ b/onadata/apps/logger/admin.py @@ -2,7 +2,11 @@ """ Logger admin module. """ -from django.contrib import admin + +from django.contrib import admin, messages +from django.core.management import call_command +from django.core.management.base import CommandError +from django.utils.translation import gettext_lazy as _ from reversion.admin import VersionAdmin @@ -12,7 +16,7 @@ class FilterByUserMixin: # pylint: disable=too-few-public-methods """Filter queryset by ``request.user``.""" - # A user should only see forms/projects that belong to him. + # A user should only see forms/projects that belong to them. def get_queryset(self, request): """Returns queryset filtered by the `request.user`.""" queryset = super().get_queryset(request) @@ -21,13 +25,60 @@ def get_queryset(self, request): return queryset.filter(**{self.user_lookup_field: request.user}) -class XFormAdmin(FilterByUserMixin, VersionAdmin, admin.ModelAdmin): +class XFormAdmin(FilterByUserMixin, admin.ModelAdmin): """Customise the XForm admin view.""" exclude = ("user",) - list_display = ("id_string", "downloadable", "shared") - search_fields = ("id_string", "title") + list_display = ("internal_id", "id_string", "project_id", "downloadable", "shared") + search_fields = ("id", "id_string", "title", "project__id", "project__name") user_lookup_field = "user" + actions = ["restore_form"] + + def internal_id(self, obj): + """Display the internal ID.""" + return obj.id + + internal_id.short_description = "Internal ID" # Label for the admin column + + def restore_form(self, request, queryset): + """Custom admin action to restore soft-deleted XForms.""" + restored_count = 0 + + for xform in queryset.iterator(chunk_size=100): + if xform.deleted_at is not None: + try: + call_command("restore_form", xform.id) + restored_count += 1 + except CommandError as exc: + self.message_user( + request, + _(f"Failed to restore XForm {xform.id_string}: {exc}"), + level=messages.ERROR, + ) + + if restored_count > 0: + self.message_user( + request, + _(f"Successfully restored {restored_count} XForms."), + level=messages.SUCCESS, + ) + + restore_form.short_description = _("Restore selected deleted XForms") + + def delete_queryset(self, request, queryset): + """ + Override delete_queryset to perform soft deletion on XForms. + """ + for xform in queryset.iterator(chunk_size=100): + if xform.deleted_at is None: + xform.soft_delete(user=request.user) + + def delete_model(self, request, obj): + """ + Override delete_model to perform soft deletion on a single XForm. + """ + if obj.deleted_at is None: + obj.soft_delete(user=request.user) admin.site.register(XForm, XFormAdmin) diff --git a/onadata/apps/logger/management/commands/restore_form.py b/onadata/apps/logger/management/commands/restore_form.py new file mode 100644 index 0000000000..fbcb5d917c --- /dev/null +++ b/onadata/apps/logger/management/commands/restore_form.py @@ -0,0 +1,58 @@ +""" +Restore a soft-deleted XForm object. +""" + +from django.core.management.base import BaseCommand, CommandError + +from onadata.apps.logger.models import XForm + + +class Command(BaseCommand): + """ + Management command to restore soft-deleted XForm objects. + + Usage: + python manage.py restore_form + """ + + help = "Restores a soft-deleted XForm." + + def add_arguments(self, parser): + # Add an argument to specify the form ID + parser.add_argument( + "form_id", + type=int, + help="The ID of the soft-deleted form to restore", + ) + + def handle(self, *args, **options): + form_id = options["form_id"] + + try: + # Retrieve the soft-deleted form + xform = XForm.objects.get(pk=form_id) + + if xform.deleted_at is None: + raise CommandError(f"Form with ID {form_id} is not soft-deleted") + + # Perform the restoration + self.stdout.write(f"Restoring form with ID {form_id}...") + was_deleted_by = xform.deleted_by.username if xform.deleted_by else None + # was_deleted_at in the format Nov. 1, 2021, HH:MM UTC + was_deleted_at = xform.deleted_at.strftime("%b. %d, %Y, %H:%M UTC") + xform.restore() + + # Display success message + success_msg = ( + f"Successfully restored form '{xform.id_string}' with " + f"ID {form_id} deleted by {was_deleted_by} at {was_deleted_at}." + ) + self.stdout.write(self.style.SUCCESS(success_msg)) + + except XForm.DoesNotExist as exc: + raise CommandError(f"Form with ID {form_id} does not exist") from exc + + except Exception as exc: + raise CommandError( + f"An error occurred while restoring the form: {exc}" + ) from exc diff --git a/onadata/apps/logger/models/data_view.py b/onadata/apps/logger/models/data_view.py index 51278f4f25..515ebdd57a 100644 --- a/onadata/apps/logger/models/data_view.py +++ b/onadata/apps/logger/models/data_view.py @@ -2,6 +2,7 @@ """ DataView model class """ + import datetime import json @@ -9,9 +10,9 @@ from django.contrib.gis.db import models from django.db import connection from django.db.models.signals import post_delete, post_save +from django.db.utils import DataError from django.utils import timezone from django.utils.translation import gettext as _ -from django.db.utils import DataError from onadata.apps.viewer.parsed_instance_tools import get_where_clause from onadata.libs.models.sorting import ( # noqa pylint: disable=unused-import @@ -22,8 +23,8 @@ from onadata.libs.utils.cache_tools import ( # noqa pylint: disable=unused-import DATAVIEW_COUNT, DATAVIEW_LAST_SUBMISSION_TIME, - XFORM_LINKED_DATAVIEWS, PROJ_OWNER_CACHE, + XFORM_LINKED_DATAVIEWS, safe_delete, ) from onadata.libs.utils.common_tags import ( @@ -216,6 +217,19 @@ def soft_delete(self, user=None): update_fields.append("deleted_by") self.save(update_fields=update_fields) + def restore(self): + """ + Restore the dataview by removing the timestamped suffix from the name + and setting the deleted_at field to None. + """ + if self.deleted_at is not None: + self.name = self.name.split("-deleted-at-")[0] + self.deleted_at = None + self.deleted_by = None + self.save( + update_fields=["name", "deleted_at", "deleted_by", "date_modified"] + ) + @classmethod def _get_where_clause( # pylint: disable=too-many-locals cls, diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index f8e7a09e50..1b0a14e3b3 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -1106,6 +1106,9 @@ def soft_delete(self, user=None): without violating the uniqueness constraint. Also soft deletes associated dataviews """ + if self.deleted_at is not None: + return + soft_deletion_time = timezone.now() deletion_suffix = soft_deletion_time.strftime("-deleted-at-%s") self.deleted_at = soft_deletion_time @@ -1140,6 +1143,38 @@ def soft_delete(self, user=None): metadata.soft_delete() clear_project_cache(self.project_id) + @transaction.atomic() + def restore(self): + """Restore a soft-deleted XForm""" + if self.deleted_at is None: + return + + self.deleted_at = None + self.id_string = self.id_string.split("-deleted-at-")[0] + self.sms_id_string = self.sms_id_string.split("-deleted-at-")[0] + self.downloadable = True + self.deleted_by = None + self.save( + update_fields=[ + "deleted_at", + "id_string", + "sms_id_string", + "downloadable", + "deleted_by", + ] + ) + # Restore associated filtered datasets + for dataview in self.dataview_set.all(): + dataview.restore() + # Restore associated Merged-Datasets + for merged_dataset in self.mergedxform_ptr.filter(deleted_at__isnull=False): + merged_dataset.restore() + # Restore associated Form Media Files + for metadata in self.metadata_set.filter(deleted_at__isnull=False): + metadata.restore() + + clear_project_cache(self.project_id) + def submission_count(self, force_update=False): """Returns the form's number of submission.""" if self.num_of_submissions == 0 or force_update: diff --git a/onadata/apps/logger/tests/models/test_data_view.py b/onadata/apps/logger/tests/models/test_data_view.py index 573e0c4358..d2daf0fddd 100644 --- a/onadata/apps/logger/tests/models/test_data_view.py +++ b/onadata/apps/logger/tests/models/test_data_view.py @@ -1,11 +1,13 @@ import os from builtins import str + from django.conf import settings from django.db import connection -from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet -from onadata.apps.logger.models.data_view import append_where_list, DataView +from onadata.apps.logger.models.data_view import DataView, append_where_list +from onadata.apps.logger.models.xform import XForm +from onadata.apps.main.tests.test_base import TestBase class TestDataView(TestBase): @@ -19,6 +21,30 @@ def test_append_where_list(self): self.assertEqual(append_where_list(">=", [], json_str), ["json->>%s >= %s"]) self.assertEqual(append_where_list("<=", [], json_str), ["json->>%s <= %s"]) + def test_restore_deleted(self): + """Soft deleted DataView can be restored""" + self._publish_transportation_form_and_submit_instance() + xform = XForm.objects.get(pk=self.xform.id) + # Create dataview for form and soft delete it + data_view = DataView.objects.create( + name="test_view", + project=self.project, + xform=xform, + columns=["name", "age"], + ) + data_view.soft_delete() + data_view.refresh_from_db() + + self.assertIsNotNone(data_view.deleted_at) + self.assertIn("-deleted-at-", data_view.name) + + # Restore DataView + data_view.restore() + data_view.refresh_from_db() + + self.assertIsNone(data_view.deleted_at) + self.assertNotIn("-deleted-at-", data_view.name) + class TestIntegratedDataView(TestAbstractViewSet): def setUp(self): diff --git a/onadata/apps/logger/tests/models/test_xform.py b/onadata/apps/logger/tests/models/test_xform.py index 52fd3c3e50..9e44e81864 100644 --- a/onadata/apps/logger/tests/models/test_xform.py +++ b/onadata/apps/logger/tests/models/test_xform.py @@ -4,13 +4,13 @@ """ import os - from builtins import str as text +from unittest.mock import call, patch -from onadata.apps.logger.models import Instance, XForm +from onadata.apps.logger.models import DataView, Instance, XForm from onadata.apps.logger.models.xform import DuplicateUUIDError, check_xform_uuid -from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.xform_instance_parser import XLSFormError +from onadata.apps.main.tests.test_base import TestBase class TestXForm(TestBase): @@ -251,3 +251,106 @@ def test_multiple_model_nodes(self): ) dd.set_uuid_in_xml() self.assertIn("\n \n \n", dd.xml) + + @patch("onadata.apps.logger.models.xform.clear_project_cache") + def test_restore_deleted(self, mock_clear_project_cache): + """Deleted XForm can be restored""" + self._publish_transportation_form_and_submit_instance() + xform = XForm.objects.get(pk=self.xform.id) + # Create dataview for form + data_view = DataView.objects.create( + name="test_view", + project=self.project, + xform=xform, + columns=["name", "age"], + ) + # Create metadata for form + metadata = xform.metadata_set.create( + data_value="test", + data_type="test", + data_file="test", + data_file_type="test", + ) + xform.soft_delete(self.user) + xform.refresh_from_db() + data_view.refresh_from_db() + metadata.refresh_from_db() + + # deleted_at is not None + self.assertIsNotNone(xform.deleted_at) + self.assertIsNotNone(data_view.deleted_at) + self.assertIsNotNone(metadata.deleted_at) + + # is inactive, no submissions will be allowed + self.assertFalse(xform.downloadable) + + # deleted-at suffix is present + self.assertIn("-deleted-at-", xform.id_string) + self.assertIn("-deleted-at-", xform.sms_id_string) + self.assertEqual(xform.deleted_by.username, "bob") + calls = [call(self.project.pk), call(self.project.pk)] + mock_clear_project_cache.has_calls(calls, any_order=True) + mock_clear_project_cache.reset_mock() + + xform.restore() + xform.refresh_from_db() + data_view.refresh_from_db() + metadata.refresh_from_db() + + # deleted_at is None + self.assertIsNone(xform.deleted_at) + self.assertIsNone(data_view.deleted_at) + self.assertIsNone(metadata.deleted_at) + + # is active + self.assertTrue(xform.downloadable) + + # deleted-at suffix not present + self.assertNotIn("-deleted-at-", xform.id_string) + self.assertNotIn("-deleted-at-", xform.sms_id_string) + self.assertIsNone(xform.deleted_by) + calls = [call(self.project.pk), call(self.project.pk)] + mock_clear_project_cache.has_calls(calls, any_order=True) + + @patch("onadata.apps.logger.models.xform.clear_project_cache") + def test_restore_deleted_merged_xform(self, mock_clear_project_cache): + """Deleted merged XForm can be restored""" + merged_xf = self._create_merged_dataset() + xform = XForm.objects.get(pk=merged_xf.pk) + + xform.soft_delete(self.user) + xform.refresh_from_db() + merged_xf.refresh_from_db() + + # deleted_at is not None + self.assertIsNotNone(xform.deleted_at) + self.assertIsNotNone(merged_xf.deleted_at) + + # is inactive, no submissions will be allowed + self.assertFalse(xform.downloadable) + + # deleted-at suffix is present + self.assertIn("-deleted-at-", xform.id_string) + self.assertIn("-deleted-at-", xform.sms_id_string) + self.assertEqual(xform.deleted_by.username, "bob") + calls = [call(self.project.pk), call(self.project.pk)] + mock_clear_project_cache.has_calls(calls, any_order=True) + mock_clear_project_cache.reset_mock() + + xform.restore() + xform.refresh_from_db() + merged_xf.refresh_from_db() + + # deleted_at is None + self.assertIsNone(xform.deleted_at) + self.assertIsNone(merged_xf.deleted_at) + + # is active + self.assertTrue(xform.downloadable) + + # deleted-at suffix not present + self.assertNotIn("-deleted-at-", xform.id_string) + self.assertNotIn("-deleted-at-", xform.sms_id_string) + self.assertIsNone(xform.deleted_by) + calls = [call(self.project.pk), call(self.project.pk)] + mock_clear_project_cache.has_calls(calls, any_order=True) diff --git a/onadata/apps/main/models/meta_data.py b/onadata/apps/main/models/meta_data.py index ed9ab47875..b8f0f61cbf 100644 --- a/onadata/apps/main/models/meta_data.py +++ b/onadata/apps/main/models/meta_data.py @@ -2,13 +2,14 @@ """ MetaData model """ + from __future__ import unicode_literals +import hashlib import logging import mimetypes import os from contextlib import closing -import hashlib from django.conf import settings from django.contrib.contenttypes.fields import GenericForeignKey @@ -24,8 +25,8 @@ import requests from onadata.libs.utils.cache_tools import ( - XFORM_METADATA_CACHE, XFORM_MANIFEST_CACHE, + XFORM_METADATA_CACHE, safe_delete, ) from onadata.libs.utils.common_tags import ( @@ -262,6 +263,13 @@ def soft_delete(self): self.deleted_at = soft_deletion_time self.save() + def restore(self): + """ + Restore the MetaData by setting the deleted_at field to None. + """ + self.deleted_at = None + self.save() + @staticmethod def public_link(content_object, data_value=None): """Returns the public link metadata.""" diff --git a/onadata/apps/main/tests/test_metadata.py b/onadata/apps/main/tests/test_metadata.py index 5f3b80c1b7..fdce0df421 100644 --- a/onadata/apps/main/tests/test_metadata.py +++ b/onadata/apps/main/tests/test_metadata.py @@ -2,6 +2,7 @@ """ Test MetaData model. """ + from django.core.cache import cache from onadata.apps.logger.models import Instance, Project, XForm @@ -180,3 +181,18 @@ def test_caches_cleared(self): metadata.save() self.assertIsNone(cache.get(key_2)) + + def test_restore_deleted(self): + """Soft deleted MetaData is restored""" + # Create metadata and soft delete + metadata = MetaData.objects.create(data_type="media", object_id=self.xform.id) + metadata.soft_delete() + metadata.refresh_from_db() + + self.assertIsNotNone(metadata.deleted_at) + + # Restore metadata + metadata.restore() + metadata.refresh_from_db() + + self.assertIsNone(metadata.deleted_at)