Skip to content

Commit fd2297d

Browse files
PatrickCmdPatrickCmd
committed
OpenConceptLab/ocl_issues#22 | Retired concepts and/or mappings are now excluded from exports
1 parent fac3d00 commit fd2297d

8 files changed

Lines changed: 161 additions & 29 deletions

File tree

‎core/collections/views.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
CollectionVersionSummaryDetailSerializer, CollectionReferenceDetailSerializer)
3232
from core.collections.utils import is_version_specified
3333
from core.common.constants import (
34-
HEAD, RELEASED_PARAM, PROCESSING_PARAM, OK_MESSAGE
34+
HEAD, RELEASED_PARAM, PROCESSING_PARAM, OK_MESSAGE, INCLUDE_RETIRED_PARAM
3535
)
3636
from core.common.mixins import (
3737
ConceptDictionaryCreateMixin, ListWithHeadersMixin, ConceptDictionaryUpdateMixin,
@@ -553,7 +553,8 @@ class CollectionVersionExportView(ConceptContainerExportMixin, CollectionVersion
553553
def handle_export_version(self):
554554
version = self.get_object()
555555
try:
556-
export_collection.delay(version.id)
556+
include_retired = parse_boolean_query_param(self.request, INCLUDE_RETIRED_PARAM, "True")
557+
export_collection.delay(version.id, include_retired)
557558
return status.HTTP_202_ACCEPTED
558559
except AlreadyQueued:
559560
return status.HTTP_409_CONFLICT

‎core/common/mixins.py‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@
1515
from rest_framework.mixins import ListModelMixin, CreateModelMixin
1616
from rest_framework.response import Response
1717

18-
from core.common.constants import HEAD, ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW, ACCESS_TYPE_NONE, INCLUDE_FACETS, \
18+
from core.common.constants import HEAD, ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW, ACCESS_TYPE_NONE, INCLUDE_FACETS, INCLUDE_RETIRED_PARAM, \
1919
LIST_DEFAULT_LIMIT, HTTP_COMPRESS_HEADER, CSV_DEFAULT_LIMIT, FACETS_ONLY, NOT_FOUND, \
2020
MUST_SPECIFY_EXTRA_PARAM_IN_BODY
2121
from core.common.permissions import HasPrivateAccess, HasOwnership, CanViewConceptDictionary
2222
from core.common.services import S3
23-
from .utils import write_csv_to_s3, get_csv_from_s3, get_query_params_from_url_string, compact_dict_by_values
23+
from .utils import write_csv_to_s3, get_csv_from_s3, get_query_params_from_url_string, compact_dict_by_values, parse_boolean_query_param
2424

2525
logger = logging.getLogger('oclapi')
2626

@@ -538,16 +538,8 @@ def get_object(self):
538538
return instance
539539

540540
def get(self, request, *args, **kwargs): # pylint: disable=unused-argument
541-
version = self.get_object()
542-
logger.debug(
543-
'Export requested for %s version %s - Requesting AWS-S3 key', self.entity.lower(), version.version
544-
)
545-
if version.is_head and not request.user.is_staff:
546-
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
547-
548-
if version.has_export():
549-
export_url = version.get_export_url()
550541

542+
def export_response():
551543
no_redirect = request.query_params.get('noRedirect', False) in ['true', 'True', True]
552544
if no_redirect:
553545
return Response(dict(url=export_url), status=status.HTTP_200_OK)
@@ -563,6 +555,24 @@ def get(self, request, *args, **kwargs): # pylint: disable=unused-argument
563555
response['Last-Updated-Timezone'] = settings.TIME_ZONE_PLACE
564556
return response
565557

558+
version = self.get_object()
559+
include_retired = parse_boolean_query_param(request, INCLUDE_RETIRED_PARAM, "True")
560+
logger.debug(
561+
'Export requested for %s version %s - Requesting AWS-S3 key', self.entity.lower(), version.version
562+
)
563+
564+
if version.is_head and not request.user.is_staff:
565+
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
566+
567+
if include_retired:
568+
if version.has_export():
569+
export_url = version.get_export_url()
570+
return export_response()
571+
else:
572+
if version.has_unretired_export():
573+
export_url = version.get_unretired_export_url()
574+
return export_response()
575+
566576
if version.is_exporting:
567577
return Response(status=status.HTTP_208_ALREADY_REPORTED)
568578

‎core/common/models.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,11 @@ def is_exporting(self):
728728
def export_path(self):
729729
last_update = self.last_child_update.strftime('%Y%m%d%H%M%S')
730730
return self.generic_export_path(suffix="{}.zip".format(last_update))
731+
732+
@cached_property
733+
def exclude_retired_export_path(self):
734+
last_update = self.last_child_update.strftime('%Y%m%d%H%M%S')
735+
return self.generic_export_path(suffix="{}_unretired.zip".format(last_update))
731736

732737
def generic_export_path(self, suffix='*'):
733738
path = "{}/{}_{}.".format(self.parent_resource, self.mnemonic, self.version)
@@ -739,9 +744,15 @@ def generic_export_path(self, suffix='*'):
739744
def get_export_url(self):
740745
return S3.url_for(self.export_path)
741746

747+
def get_unretired_export_url(self):
748+
return S3.url_for(self.exclude_retired_export_path)
749+
742750
def has_export(self):
743751
return S3.exists(self.export_path)
744752

753+
def has_unretired_export(self):
754+
return S3.exists(self.exclude_retired_export_path)
755+
745756

746757
class CelerySignalProcessor(RealTimeSignalProcessor):
747758
def handle_save(self, sender, instance, **kwargs):

‎core/common/tasks.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def delete_organization(org_id):
3737

3838

3939
@app.task(base=QueueOnce, bind=True)
40-
def export_source(self, version_id):
40+
def export_source(self, version_id, include_retired=True):
4141
from core.sources.models import Source
4242
logger.info('Finding source version...')
4343

@@ -52,14 +52,14 @@ def export_source(self, version_id):
5252
version.add_processing(self.request.id)
5353
try:
5454
logger.info('Found source version %s. Beginning export...', version.version)
55-
write_export_file(version, 'source', 'core.sources.serializers.SourceVersionExportSerializer', logger)
55+
write_export_file(version, include_retired, 'source', 'core.sources.serializers.SourceVersionExportSerializer', logger)
5656
logger.info('Export complete!')
5757
finally:
5858
version.remove_processing(self.request.id)
5959

6060

6161
@app.task(base=QueueOnce, bind=True)
62-
def export_collection(self, version_id):
62+
def export_collection(self, version_id, include_retired=True):
6363
from core.collections.models import Collection
6464
logger.info('Finding collection version...')
6565

@@ -75,7 +75,7 @@ def export_collection(self, version_id):
7575
try:
7676
logger.info('Found collection version %s. Beginning export...', version.version)
7777
write_export_file(
78-
version, 'collection', 'core.collections.serializers.CollectionVersionExportSerializer', logger
78+
version, include_retired, 'collection', 'core.collections.serializers.CollectionVersionExportSerializer', logger
7979
)
8080
logger.info('Export complete!')
8181
finally:

‎core/common/utils.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ def get_class(kls):
191191

192192

193193
def write_export_file(
194-
version, resource_type, resource_serializer_type, logger
194+
version, include_retired, resource_type, resource_serializer_type, logger
195195
): # pylint: disable=too-many-statements,too-many-locals,too-many-branches
196196
from core.concepts.models import Concept
197197
from core.mappings.models import Mapping
@@ -207,6 +207,12 @@ def write_export_file(
207207
logger.info('Done serializing attributes.')
208208

209209
batch_size = 100
210+
concepts_qs = version.concepts
211+
mappings_qs = version.mappings
212+
if not include_retired:
213+
concepts_qs = concepts_qs.filter(retired=False)
214+
mappings_qs = mappings_qs.filter(retired=False)
215+
210216
is_collection = resource_type == 'collection'
211217

212218
if is_collection:
@@ -329,7 +335,7 @@ def write_export_file(
329335
logger.info(file_path)
330336
logger.info('Done compressing. Uploading...')
331337

332-
s3_key = version.export_path
338+
s3_key = version.export_path if include_retired else version.exclude_retired_export_path
333339
S3.upload_file(
334340
key=s3_key, file_path=file_path, binary=True, metadata=dict(ContentType='application/zip'),
335341
headers={'content-type': 'application/zip'}

‎core/integration_tests/tests_collections.py‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -875,29 +875,29 @@ def test_post_303(self, s3_exists_mock):
875875
def test_post_202(self, s3_exists_mock, export_collection_mock):
876876
s3_exists_mock.return_value = False
877877
response = self.client.post(
878-
'/collections/coll/v1/export/',
878+
'/collections/coll/v1/export/?includeRetired=False',
879879
HTTP_AUTHORIZATION='Token ' + self.token,
880880
format='json'
881881
)
882882

883883
self.assertEqual(response.status_code, 202)
884884
s3_exists_mock.assert_called_once_with("username/coll_v1.{}.zip".format(self.v1_updated_at))
885-
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id)
885+
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id, False)
886886

887887
@patch('core.collections.views.export_collection')
888888
@patch('core.common.services.S3.exists')
889889
def test_post_409(self, s3_exists_mock, export_collection_mock):
890890
s3_exists_mock.return_value = False
891891
export_collection_mock.delay.side_effect = AlreadyQueued('already-queued')
892892
response = self.client.post(
893-
'/collections/coll/v1/export/',
893+
'/collections/coll/v1/export/?includeRetired=False',
894894
HTTP_AUTHORIZATION='Token ' + self.token,
895895
format='json'
896896
)
897897

898898
self.assertEqual(response.status_code, 409)
899899
s3_exists_mock.assert_called_once_with("username/coll_v1.{}.zip".format(self.v1_updated_at))
900-
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id)
900+
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id, False)
901901

902902

903903
class CollectionVersionListViewTest(OCLAPITestCase):
@@ -1017,6 +1017,62 @@ def test_export_collection(self, s3_mock): # pylint: disable=too-many-locals
10171017
import shutil
10181018
shutil.rmtree(latest_temp_dir)
10191019

1020+
@patch('core.common.utils.S3')
1021+
def test_unretired_export_collection(self, s3_mock): # pylint: disable=too-many-locals
1022+
s3_mock.url_for = Mock(return_value='https://s3-url')
1023+
s3_mock.upload_file = Mock()
1024+
source = OrganizationSourceFactory()
1025+
concept1 = ConceptFactory(parent=source)
1026+
concept2 = ConceptFactory(parent=source)
1027+
mapping = MappingFactory(from_concept=concept2, to_concept=concept1, parent=source)
1028+
collection = OrganizationCollectionFactory()
1029+
collection.add_references([concept1.uri, concept2.uri, mapping.uri])
1030+
collection.refresh_from_db()
1031+
1032+
export_collection(collection.id, include_retired=False) # pylint: disable=no-value-for-parameter
1033+
1034+
latest_temp_dir = get_latest_dir_in_path('/tmp/')
1035+
zipped_file = zipfile.ZipFile(latest_temp_dir + '/export.zip')
1036+
exported_data = json.loads(zipped_file.read('export.json').decode('utf-8'))
1037+
1038+
self.assertEqual(
1039+
exported_data,
1040+
{**CollectionVersionExportSerializer(collection).data, 'concepts': ANY, 'mappings': ANY, 'references': ANY}
1041+
)
1042+
1043+
exported_concepts = exported_data['concepts']
1044+
expected_concepts = ConceptVersionExportSerializer(
1045+
[concept2.get_latest_version(), concept1.get_latest_version()], many=True
1046+
).data
1047+
1048+
self.assertEqual(len(exported_concepts), 2)
1049+
self.assertIn(expected_concepts[0], exported_concepts)
1050+
self.assertIn(expected_concepts[1], exported_concepts)
1051+
1052+
exported_mappings = exported_data['mappings']
1053+
expected_mappings = MappingDetailSerializer([mapping.get_latest_version()], many=True).data
1054+
1055+
self.assertEqual(len(exported_mappings), 1)
1056+
self.assertEqual(expected_mappings, exported_mappings)
1057+
1058+
exported_references = exported_data['references']
1059+
expected_references = CollectionReferenceSerializer(collection.references.all(), many=True).data
1060+
1061+
self.assertEqual(len(exported_references), 3)
1062+
self.assertIn(exported_references[0], expected_references)
1063+
self.assertIn(exported_references[1], expected_references)
1064+
self.assertIn(exported_references[2], expected_references)
1065+
1066+
s3_upload_key = collection.exclude_retired_export_path
1067+
s3_mock.upload_file.assert_called_once_with(
1068+
key=s3_upload_key, file_path=latest_temp_dir + '/export.zip', binary=True,
1069+
metadata={'ContentType': 'application/zip'}, headers={'content-type': 'application/zip'}
1070+
)
1071+
s3_mock.url_for.assert_called_once_with(s3_upload_key)
1072+
1073+
import shutil
1074+
shutil.rmtree(latest_temp_dir)
1075+
10201076

10211077
class CollectionConceptsViewTest(OCLAPITestCase):
10221078
def setUp(self):

‎core/integration_tests/tests_sources.py‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -705,29 +705,29 @@ def test_post_303(self, s3_exists_mock):
705705
def test_post_202(self, s3_exists_mock, export_source_mock):
706706
s3_exists_mock.return_value = False
707707
response = self.client.post(
708-
'/sources/source1/v1/export/',
708+
'/sources/source1/v1/export/?includeRetired=False',
709709
HTTP_AUTHORIZATION='Token ' + self.token,
710710
format='json'
711711
)
712712

713713
self.assertEqual(response.status_code, 202)
714714
s3_exists_mock.assert_called_once_with("username/source1_v1.{}.zip".format(self.v1_updated_at))
715-
export_source_mock.delay.assert_called_once_with(self.source_v1.id)
715+
export_source_mock.delay.assert_called_once_with(self.source_v1.id, False)
716716

717717
@patch('core.sources.views.export_source')
718718
@patch('core.common.services.S3.exists')
719719
def test_post_409(self, s3_exists_mock, export_source_mock):
720720
s3_exists_mock.return_value = False
721721
export_source_mock.delay.side_effect = AlreadyQueued('already-queued')
722722
response = self.client.post(
723-
'/sources/source1/v1/export/',
723+
'/sources/source1/v1/export/?includeRetired=False',
724724
HTTP_AUTHORIZATION='Token ' + self.token,
725725
format='json'
726726
)
727727

728728
self.assertEqual(response.status_code, 409)
729729
s3_exists_mock.assert_called_once_with("username/source1_v1.{}.zip".format(self.v1_updated_at))
730-
export_source_mock.delay.assert_called_once_with(self.source_v1.id)
730+
export_source_mock.delay.assert_called_once_with(self.source_v1.id, False)
731731

732732

733733
class ExportSourceTaskTest(OCLAPITestCase):
@@ -778,6 +778,53 @@ def test_export_source(self, s3_mock): # pylint: disable=too-many-locals
778778
import shutil
779779
shutil.rmtree(latest_temp_dir)
780780

781+
@patch('core.common.utils.S3')
782+
def test_unretired_export_source(self, s3_mock): # pylint: disable=too-many-locals
783+
s3_mock.url_for = Mock(return_value='https://s3-url')
784+
s3_mock.upload_file = Mock()
785+
source = OrganizationSourceFactory()
786+
concept1 = ConceptFactory(parent=source)
787+
concept2 = ConceptFactory(parent=source)
788+
mapping = MappingFactory(from_concept=concept2, to_concept=concept1, parent=source)
789+
790+
source_v1 = OrganizationSourceFactory(mnemonic=source.mnemonic, organization=source.organization, version='v1')
791+
concept1.sources.add(source_v1)
792+
concept2.sources.add(source_v1)
793+
mapping.sources.add(source_v1)
794+
795+
export_source(source_v1.id, include_retired=False) # pylint: disable=no-value-for-parameter
796+
797+
latest_temp_dir = get_latest_dir_in_path('/tmp/')
798+
zipped_file = zipfile.ZipFile(latest_temp_dir + '/export.zip')
799+
exported_data = json.loads(zipped_file.read('export.json').decode('utf-8'))
800+
801+
self.assertEqual(
802+
exported_data, {**SourceVersionExportSerializer(source_v1).data, 'concepts': ANY, 'mappings': ANY}
803+
)
804+
805+
exported_concepts = exported_data['concepts']
806+
expected_concepts = ConceptVersionExportSerializer([concept2, concept1], many=True).data
807+
808+
self.assertEqual(len(exported_concepts), 2)
809+
self.assertIn(expected_concepts[0], exported_concepts)
810+
self.assertIn(expected_concepts[1], exported_concepts)
811+
812+
exported_mappings = exported_data['mappings']
813+
expected_mappings = MappingDetailSerializer([mapping], many=True).data
814+
815+
self.assertEqual(len(exported_mappings), 1)
816+
self.assertEqual(expected_mappings, exported_mappings)
817+
818+
s3_upload_key = source_v1.exclude_retired_export_path
819+
s3_mock.upload_file.assert_called_once_with(
820+
key=s3_upload_key, file_path=latest_temp_dir + '/export.zip', binary=True,
821+
metadata={'ContentType': 'application/zip'}, headers={'content-type': 'application/zip'}
822+
)
823+
s3_mock.url_for.assert_called_once_with(s3_upload_key)
824+
825+
import shutil
826+
shutil.rmtree(latest_temp_dir)
827+
781828

782829
class SourceLogoViewTest(OCLAPITestCase):
783830
def setUp(self):

‎core/sources/views.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from rest_framework.response import Response
1515

1616
from core.client_configs.views import ResourceClientConfigsView
17-
from core.common.constants import HEAD, RELEASED_PARAM, PROCESSING_PARAM
17+
from core.common.constants import HEAD, RELEASED_PARAM, PROCESSING_PARAM, INCLUDE_RETIRED_PARAM
1818
from core.common.mixins import ListWithHeadersMixin, ConceptDictionaryCreateMixin, ConceptDictionaryUpdateMixin, \
1919
ConceptContainerExportMixin, ConceptContainerProcessingMixin, ConceptContainerExtraRetrieveUpdateDestroyView
2020
from core.common.permissions import CanViewConceptDictionary, CanEditConceptDictionary, HasAccessToVersionedObject, \
@@ -323,7 +323,8 @@ class SourceVersionExportView(ConceptContainerExportMixin, SourceVersionBaseView
323323
def handle_export_version(self):
324324
version = self.get_object()
325325
try:
326-
export_source.delay(version.id)
326+
include_retired = parse_boolean_query_param(self.request, INCLUDE_RETIRED_PARAM, "True")
327+
export_source.delay(version.id, include_retired)
327328
return status.HTTP_202_ACCEPTED
328329
except AlreadyQueued:
329330
return status.HTTP_409_CONFLICT

0 commit comments

Comments
 (0)