From f5f236ad1250bf8fdc7c5b50de3f7ca93830ea7c Mon Sep 17 00:00:00 2001 From: Grant Gainey Date: Wed, 27 Aug 2025 07:54:40 -0400 Subject: [PATCH] [PULP-747] Made pulp/fs import-export work in the presence of domains. Added tests for fs-export. closes #6926. --- CHANGES/6926.feature | 3 + docs/user/guides/create-domains.md | 4 - .../functional/api/test_filesystem_export.py | 220 ++++++++++++++++++ .../tests/functional/api/test_pulp_export.py | 106 ++++++++- pulpcore/app/importexport.py | 20 +- .../0145_domainize_import_export.py | 53 +++++ pulpcore/app/modelresource.py | 81 +++++-- pulpcore/app/models/exporter.py | 8 +- pulpcore/app/models/fields.py | 1 - pulpcore/app/models/importer.py | 9 +- pulpcore/app/models/repository.py | 1 + pulpcore/app/serializers/exporter.py | 8 +- pulpcore/app/serializers/importer.py | 4 +- pulpcore/app/tasks/importer.py | 52 ++++- pulpcore/app/viewsets/exporter.py | 9 +- pulpcore/app/viewsets/importer.py | 7 +- 16 files changed, 523 insertions(+), 63 deletions(-) create mode 100644 CHANGES/6926.feature create mode 100644 pulp_file/tests/functional/api/test_filesystem_export.py create mode 100644 pulpcore/app/migrations/0145_domainize_import_export.py diff --git a/CHANGES/6926.feature b/CHANGES/6926.feature new file mode 100644 index 0000000000..bc43a61515 --- /dev/null +++ b/CHANGES/6926.feature @@ -0,0 +1,3 @@ +Taught pulp-import-export to work in a domain-enabled environment. + +All combinations of domain-state between upstream and downstream are handled. diff --git a/docs/user/guides/create-domains.md b/docs/user/guides/create-domains.md index ba544d0265..1ce161fc9f 100644 --- a/docs/user/guides/create-domains.md +++ b/docs/user/guides/create-domains.md @@ -133,10 +133,6 @@ pulp user role-assignment add --username --role --domain e.g. You can not add content from one domain to the repository of another domain even if you own both domains. -!!! warning - Pulp Export and Import are currently not supported with domains enabled. - - There are notable objects in Pulp like `Roles`, `Users`, and `Groups`, that are not a part of domains and remain global across the system. These objects are closely intertwined with the RBAC system and currently do not make sense to be unique on the domain level. Objects that are not a part of a domain are readable from any domain (with the correct permissions), diff --git a/pulp_file/tests/functional/api/test_filesystem_export.py b/pulp_file/tests/functional/api/test_filesystem_export.py new file mode 100644 index 0000000000..053f656f23 --- /dev/null +++ b/pulp_file/tests/functional/api/test_filesystem_export.py @@ -0,0 +1,220 @@ +import json +import pytest +import uuid + +from pulpcore.client.pulpcore.exceptions import ApiException, BadRequestException +from pulpcore.app import settings +from pulpcore.constants import TASK_STATES + + +pytestmark = [ + pytest.mark.skipif( + "/tmp" not in settings.ALLOWED_EXPORT_PATHS, + reason="Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS " + f"({settings.ALLOWED_EXPORT_PATHS}).", + ), +] + + +@pytest.fixture +def fs_exporter_factory( + tmpdir, + pulpcore_bindings, + gen_object_with_cleanup, + add_to_filesystem_cleanup, +): + def _fs_exporter_factory(method="write", pulp_domain=None): + name = str(uuid.uuid4()) + path = "{}/{}/".format(tmpdir, name) + body = { + "name": name, + "path": path, + "method": method, + } + kwargs = {} + if pulp_domain: + kwargs["pulp_domain"] = pulp_domain + exporter = gen_object_with_cleanup(pulpcore_bindings.ExportersFilesystemApi, body, **kwargs) + add_to_filesystem_cleanup(path) + assert exporter.name == name + assert exporter.path == path + assert exporter.method == method + return exporter + + return _fs_exporter_factory + + +@pytest.fixture +def fs_export_factory(pulpcore_bindings, monitor_task): + def _fs_export_factory(exporter, body): + task = monitor_task( + pulpcore_bindings.ExportersFilesystemExportsApi.create( + exporter.pulp_href, body or {} + ).task + ) + assert len(task.created_resources) == 1 + export = pulpcore_bindings.ExportersFilesystemExportsApi.read(task.created_resources[0]) + for report in task.progress_reports: + assert report.state == TASK_STATES.COMPLETED + return export + + return _fs_export_factory + + +@pytest.fixture +def pub_and_repo( + file_repository_factory, + file_bindings, + gen_object_with_cleanup, + random_artifact_factory, + monitor_task, +): + def _pub_and_repo(pulp_domain=None): + random_artifact = random_artifact_factory(pulp_domain=pulp_domain) + repository = file_repository_factory(pulp_domain=pulp_domain) + kwargs = {} + if pulp_domain: + kwargs["pulp_domain"] = pulp_domain + for i in range(2): + monitor_task( + file_bindings.ContentFilesApi.create( + artifact=random_artifact.pulp_href, + relative_path=f"{i}.dat", + repository=repository.pulp_href, + **kwargs, + ).task + ) + publish_data = file_bindings.FileFilePublication(repository=repository.pulp_href) + publication = gen_object_with_cleanup( + file_bindings.PublicationsFileApi, publish_data, **kwargs + ) + return publication, repository + + return _pub_and_repo + + +@pytest.mark.parallel +def test_crud_fsexporter(fs_exporter_factory, pulpcore_bindings, monitor_task): + # READ + exporter = fs_exporter_factory() + exporter_read = pulpcore_bindings.ExportersFilesystemApi.read(exporter.pulp_href) + assert exporter_read.name == exporter.name + assert exporter_read.path == exporter.path + + # UPDATE + body = {"path": "/tmp/{}".format(str(uuid.uuid4()))} + result = pulpcore_bindings.ExportersFilesystemApi.partial_update(exporter.pulp_href, body) + monitor_task(result.task) + exporter_read = pulpcore_bindings.ExportersFilesystemApi.read(exporter.pulp_href) + assert exporter_read.path != exporter.path + assert exporter_read.path == body["path"] + + # LIST + exporters = pulpcore_bindings.ExportersFilesystemApi.list(name=exporter.name).results + assert exporter.name in [e.name for e in exporters] + + # DELETE + result = pulpcore_bindings.ExportersFilesystemApi.delete(exporter.pulp_href) + monitor_task(result.task) + with pytest.raises(ApiException): + pulpcore_bindings.ExportersFilesystemApi.read(exporter.pulp_href) + + +@pytest.mark.parallel +def test_fsexport(pulpcore_bindings, fs_exporter_factory, fs_export_factory, pub_and_repo): + exporter = fs_exporter_factory() + (publication, _) = pub_and_repo() + # Test export + body = {"publication": publication.pulp_href} + export = fs_export_factory(exporter, body=body) + + # Test list and delete + exports = pulpcore_bindings.ExportersPulpExportsApi.list(exporter.pulp_href).results + assert len(exports) == 1 + pulpcore_bindings.ExportersPulpExportsApi.delete(export.pulp_href) + exports = pulpcore_bindings.ExportersPulpExportsApi.list(exporter.pulp_href).results + assert len(exports) == 0 + + +@pytest.mark.parallel +def test_fsexport_by_version( + fs_exporter_factory, + fs_export_factory, + pub_and_repo, +): + (publication, repository) = pub_and_repo() + latest = repository.latest_version_href + zeroth = latest.replace("/2/", "/0/") + + # export by version + exporter = fs_exporter_factory() + body = {"repository_version": latest} + fs_export_factory(exporter, body=body) + + # export by version with start_version + exporter = fs_exporter_factory() + body = {"repository_version": latest, "start_repository_version": zeroth} + fs_export_factory(exporter, body=body) + + # export by publication with start_version + exporter = fs_exporter_factory() + body = {"publication": publication.pulp_href, "start_repository_version": zeroth} + fs_export_factory(exporter, body=body) + + # negative: specify publication and version + with pytest.raises(BadRequestException) as e: + exporter = fs_exporter_factory() + body = {"publication": publication.pulp_href, "repository_version": zeroth} + fs_export_factory(exporter, body=body) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [ + "publication or repository_version must either be supplied but not both." + ] + } + + +@pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.") +@pytest.mark.parallel +def test_fsexport_cross_domain( + fs_exporter_factory, + fs_export_factory, + gen_object_with_cleanup, + pulpcore_bindings, + pub_and_repo, +): + + entities = [{}, {}] + for e in entities: + body = { + "name": str(uuid.uuid4()), + "storage_class": "pulpcore.app.models.storage.FileSystem", + "storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"}, + } + e["domain"] = gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body) + (e["publication"], e["repository"]) = pub_and_repo(pulp_domain=e["domain"].name) + e["exporter"] = fs_exporter_factory(pulp_domain=e["domain"].name) + body = {"publication": e["publication"].pulp_href} + e["export"] = fs_export_factory(e["exporter"], body=body) + + latest = entities[0]["repository"].latest_version_href + zeroth = latest.replace("/2/", "/0/") + + with pytest.raises(BadRequestException) as e: + body = {"publication": entities[0]["publication"].pulp_href} + fs_export_factory(entities[1]["exporter"], body=body) + + with pytest.raises(BadRequestException) as e: + body = {"repository_version": latest} + fs_export_factory(entities[1]["exporter"], body=body) + + with pytest.raises(BadRequestException) as e: + body = {"repository_version": latest, "start_repository_version": zeroth} + fs_export_factory(entities[1]["exporter"], body=body) + + with pytest.raises(BadRequestException) as e: + body = { + "publication": entities[0]["publication"].pulp_href, + "start_repository_version": zeroth, + } + fs_export_factory(entities[1]["exporter"], body=body) diff --git a/pulp_file/tests/functional/api/test_pulp_export.py b/pulp_file/tests/functional/api/test_pulp_export.py index c3c0907866..0991fca546 100644 --- a/pulp_file/tests/functional/api/test_pulp_export.py +++ b/pulp_file/tests/functional/api/test_pulp_export.py @@ -1,13 +1,14 @@ +import json import pytest import uuid from pulpcore.client.pulpcore.exceptions import ApiException +from pulpcore.client.pulpcore.exceptions import BadRequestException from pulpcore.app import settings from pulpcore.constants import TASK_STATES pytestmark = [ - pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export."), pytest.mark.skipif( "/tmp" not in settings.ALLOWED_EXPORT_PATHS, reason="Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS " @@ -23,7 +24,10 @@ def pulp_exporter_factory( gen_object_with_cleanup, add_to_filesystem_cleanup, ): - def _pulp_exporter_factory(repositories=None): + def _pulp_exporter_factory( + repositories=None, + pulp_domain=None, + ): if repositories is None: repositories = [] name = str(uuid.uuid4()) @@ -33,7 +37,11 @@ def _pulp_exporter_factory(repositories=None): "path": path, "repositories": [r.pulp_href for r in repositories], } - exporter = gen_object_with_cleanup(pulpcore_bindings.ExportersPulpApi, body) + kwargs = {} + if pulp_domain: + kwargs["pulp_domain"] = pulp_domain.name + + exporter = gen_object_with_cleanup(pulpcore_bindings.ExportersPulpApi, body, **kwargs) add_to_filesystem_cleanup(path) assert exporter.name == name assert exporter.path == path @@ -291,3 +299,95 @@ def test_export_incremental( with pytest.raises(ApiException): body = {"start_versions": [file_repo.latest_version_href], "full": False} pulp_export_factory(exporter, body) + + +# Test that cross-domain attempts fail +# Exporter: repository, last-export : create, update +# Export: versions, start_versions +@pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.") +@pytest.mark.parallel +def test_cross_domain_exporter( + basic_manifest_path, + file_bindings, + file_remote_factory, + gen_object_with_cleanup, + pulpcore_bindings, + pulp_export_factory, + pulp_exporter_factory, + monitor_task, +): + # Create two domains + # In each, create and sync a repository, create and export an exporter + # Attempt to create an exporter using the *other domain's* repo + # Attempt to update the exporter using the *other domain's* repo and last_export + # Use the exporter and attempt to export the *other domain's* repo-versions + + entities = [{}, {}] + for e in entities: + body = { + "name": str(uuid.uuid4()), + "storage_class": "pulpcore.app.models.storage.FileSystem", + "storage_settings": {"MEDIA_ROOT": "/var/lib/pulp/media/"}, + } + e["domain"] = gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body) + remote = file_remote_factory( + manifest_path=basic_manifest_path, policy="immediate", pulp_domain=e["domain"].name + ) + + repo_body = {"name": str(uuid.uuid4()), "remote": remote.pulp_href} + e["repository"] = gen_object_with_cleanup( + file_bindings.RepositoriesFileApi, repo_body, pulp_domain=e["domain"].name + ) + task = file_bindings.RepositoriesFileApi.sync(e["repository"].pulp_href, {}).task + monitor_task(task) + e["repository"] = file_bindings.RepositoriesFileApi.read(e["repository"].pulp_href) + e["exporter"] = pulp_exporter_factory( + repositories=[e["repository"]], pulp_domain=e["domain"] + ) + e["export"] = pulp_export_factory(e["exporter"]) + + target_domain = entities[1]["domain"] + # cross-create + with pytest.raises(BadRequestException) as e: + pulp_exporter_factory(repositories=[entities[0]["repository"]], pulp_domain=target_domain) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be a part of the {target_domain.name} domain."] + } + # cross-update + body = {"repositories": [entities[0]["repository"].pulp_href]} + with pytest.raises(BadRequestException) as e: + pulpcore_bindings.ExportersPulpApi.partial_update(entities[1]["exporter"].pulp_href, body) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be a part of the {target_domain.name} domain."] + } + + body = {"last_export": entities[0]["export"].pulp_href} + with pytest.raises(BadRequestException) as e: + pulpcore_bindings.ExportersPulpApi.partial_update(entities[1]["exporter"].pulp_href, body) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be a part of the {target_domain.name} domain."] + } + + # cross-export + with pytest.raises(BadRequestException) as e: + latest_v = entities[0]["repository"].latest_version_href + zero_v = latest_v.replace("/1/", "/0/") + body = { + "start_versions": [latest_v], + "versions": [zero_v], + "full": False, + } + pulp_export_factory(entities[1]["exporter"], body) + assert e.value.status == 400 + msgs = json.loads(e.value.body) + assert "versions" in msgs + assert "start_versions" in msgs + assert msgs["versions"] == [ + "Requested RepositoryVersions must belong to the Repositories named by the Exporter!" + ] + assert msgs["start_versions"] == [ + "Requested RepositoryVersions must belong to the Repositories named by the Exporter!" + ] diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index c8ade37077..e448dc158a 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -130,14 +130,30 @@ def export_artifacts(export, artifact_pks): temp_file.write(artifact.file.read()) temp_file.flush() artifact.file.close() - export.tarfile.add(temp_file.name, artifact.file.name) + # If we're domain-enabled, replace our domain-pk with "DOMAIN" in + # the tarfile + if settings.DOMAIN_ENABLED: + tarfile_loc = artifact.file.name.replace( + str(artifact.pulp_domain_id), "DOMAIN" + ) + else: + tarfile_loc = artifact.file.name + export.tarfile.add(temp_file.name, tarfile_loc) else: for offset in range(0, len(artifact_pks), EXPORT_BATCH_SIZE): batch = artifact_pks[offset : offset + EXPORT_BATCH_SIZE] batch_qs = Artifact.objects.filter(pk__in=batch).only("file") for artifact in pb.iter(batch_qs.iterator()): - export.tarfile.add(artifact.file.path, artifact.file.name) + # If we're domain-enabled, replace our domain-pk with "DOMAIN" in + # the tarfile + if settings.DOMAIN_ENABLED: + tarfile_loc = artifact.file.name.replace( + str(artifact.pulp_domain_id), "DOMAIN" + ) + else: + tarfile_loc = artifact.file.name + export.tarfile.add(artifact.file.path, tarfile_loc) resource = ArtifactResource() resource.queryset = Artifact.objects.filter(pk__in=artifact_pks) diff --git a/pulpcore/app/migrations/0145_domainize_import_export.py b/pulpcore/app/migrations/0145_domainize_import_export.py new file mode 100644 index 0000000000..1722f54b0a --- /dev/null +++ b/pulpcore/app/migrations/0145_domainize_import_export.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.23 on 2025-08-21 21:21 + +from django.db import migrations, models +import django.db.models.deletion +import pulpcore.app.util + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0144_delete_old_appstatus"), + ] + + operations = [ + migrations.AddField( + model_name="export", + name="pulp_domain", + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"), + ), + migrations.AddField( + model_name="exporter", + name="pulp_domain", + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"), + ), + migrations.AlterUniqueTogether( + name="exporter", + unique_together={("name", "pulp_domain")}, + ), + migrations.AddField( + model_name="import", + name="pulp_domain", + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"), + ), + migrations.AddField( + model_name="importer", + name="pulp_domain", + field=models.ForeignKey(default=pulpcore.app.util.get_domain_pk, on_delete=django.db.models.deletion.PROTECT, to="core.domain"), + ), + migrations.AlterUniqueTogether( + name="importer", + unique_together={("name", "pulp_domain")}, + ), + migrations.AlterField( + model_name="exporter", + name="name", + field=models.TextField(), + ), + migrations.AlterField( + model_name="importer", + name="name", + field=models.TextField(), + ), + ] diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py index 54d3522fa9..f4d215fc56 100644 --- a/pulpcore/app/modelresource.py +++ b/pulpcore/app/modelresource.py @@ -1,3 +1,6 @@ +import re + +from django.conf import settings from import_export import fields from import_export.widgets import ForeignKeyWidget from logging import getLogger @@ -8,12 +11,16 @@ ContentArtifact, ) from pulpcore.app.models.repository import Repository +from pulpcore.app.util import get_domain_pk from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS from pulpcore.plugin.importexport import QueryModelResource - log = getLogger(__name__) +domain_artifact_file_regex = re.compile( + r"^artifact/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", +) + # # Artifact and Repository are different from other import-export entities, in that they are not @@ -31,6 +38,24 @@ def before_import_row(self, row, **kwargs): kwargs: args passed along from the import() call. """ + # IF we're domain-enabled: + # REPLACE "their" domain-id with ours, if there is one. + # If not, INSERT "our" domain-id into the path in the right place. + # Otherwise: + # REMOVE their domain-id if there is one. + # Do this before letting QMR run, since it will replace "their" domain-id with "ours" + upstream_domain_enabled = re.match(domain_artifact_file_regex, row["file"]) + domain = str(get_domain_pk()) + + if settings.DOMAIN_ENABLED: # Replace "their" domain-id with "ours" + if upstream_domain_enabled: + row["file"] = row["file"].replace(row["pulp_domain"], domain) + else: # Add in our domain-id to the path + row["file"] = row["file"].replace("artifact", f"artifact/{domain}") + else: # Strip domain-id out of the artifact-file *if there is one there* + if upstream_domain_enabled: + row["file"] = row["file"].replace(f'artifact/{row["pulp_domain"]}/', "artifact/") + super().before_import_row(row, **kwargs) # the export converts None to blank strings but sha384 and sha512 have unique constraints @@ -39,14 +64,15 @@ def before_import_row(self, row, **kwargs): if row[checksum] == "": row[checksum] = None + def set_up_queryset(self): + """ + :return: Artifacts for a specific domain + """ + return Artifact.objects.filter(pulp_domain_id=get_domain_pk()) + class Meta: model = Artifact - exclude = ( - "pulp_id", - "pulp_created", - "pulp_last_updated", - "timestamp_of_interest", - ) + exclude = QueryModelResource.Meta.exclude + ("timestamp_of_interest",) import_id_fields = ( "pulp_domain", "sha256", @@ -54,16 +80,20 @@ class Meta: class RepositoryResource(QueryModelResource): + + def set_up_queryset(self): + """ + :return: Repositories for a specific domain + """ + return Repository.objects.filter(pulp_domain_id=get_domain_pk()) + class Meta: model = Repository import_id_fields = ( "pulp_domain", "name", ) - exclude = ( - "pulp_id", - "pulp_created", - "pulp_last_updated", + exclude = QueryModelResource.Meta.exclude + ( "content", "next_version", "repository_ptr", @@ -72,9 +102,18 @@ class Meta: ) +class ArtifactDomainForeignKeyWidget(ForeignKeyWidget): + def get_queryset(self, value, row, *args, **kwargs): + qs = self.model.objects.filter(sha256=row["artifact"], pulp_domain_id=get_domain_pk()) + return qs + + def render(self, value, obj=None, **kwargs): + return value.sha256 + + class ContentArtifactResource(QueryModelResource): """ - Handles import/export of the ContentArtifact model. + Handles import/export of the ContentArtifact model ContentArtifact is different from other import-export entities because it has no 'natural key' other than a pulp_id, which aren't shared across instances. We do some magic to link up @@ -85,7 +124,9 @@ class ContentArtifactResource(QueryModelResource): """ artifact = fields.Field( - column_name="artifact", attribute="artifact", widget=ForeignKeyWidget(Artifact, "sha256") + column_name="artifact", + attribute="artifact", + widget=ArtifactDomainForeignKeyWidget(Artifact, "sha256"), ) linked_content = {} @@ -121,18 +162,21 @@ def set_up_queryset(self): for content_ids in self.content_mapping.values(): content_pks |= set(content_ids) - return ( + qs = ( ContentArtifact.objects.filter(content__in=content_pks) .order_by("content", "relative_path") .select_related("artifact") ) + return qs def dehydrate_content(self, content_artifact): return str(content_artifact.content_id) def fetch_linked_content(self): linked_content = {} - c_qs = Content.objects.filter(upstream_id__isnull=False).values("upstream_id", "pulp_id") + c_qs = Content.objects.filter( + upstream_id__isnull=False, pulp_domain_id=get_domain_pk() + ).values("upstream_id", "pulp_id") for c in c_qs.iterator(): linked_content[str(c["upstream_id"])] = str(c["pulp_id"]) @@ -144,9 +188,4 @@ class Meta: "content", "relative_path", ) - exclude = ( - "pulp_created", - "pulp_last_updated", - "_artifacts", - "pulp_id", - ) + exclude = QueryModelResource.Meta.exclude + ("_artifacts",) diff --git a/pulpcore/app/models/exporter.py b/pulpcore/app/models/exporter.py index b6c17e8c62..37a1b1d265 100644 --- a/pulpcore/app/models/exporter.py +++ b/pulpcore/app/models/exporter.py @@ -9,6 +9,7 @@ MasterModel, ) from pulpcore.app.models.repository import Repository +from pulpcore.app.util import get_domain_pk from pulpcore.constants import FS_EXPORT_CHOICES, FS_EXPORT_METHODS @@ -29,6 +30,7 @@ class Export(BaseModel): params = models.JSONField(null=True) task = models.ForeignKey("Task", on_delete=models.SET_NULL, null=True) exporter = models.ForeignKey("Exporter", on_delete=models.CASCADE) + pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT) class ExportedResource(GenericRelationModel): @@ -54,7 +56,11 @@ class Exporter(MasterModel): name (models.TextField): The exporter unique name. """ - name = models.TextField(db_index=True, unique=True) + name = models.TextField() + pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT) + + class Meta: + unique_together = ("name", "pulp_domain") class FilesystemExport(Export): diff --git a/pulpcore/app/models/fields.py b/pulpcore/app/models/fields.py index 562d825689..11bc393056 100644 --- a/pulpcore/app/models/fields.py +++ b/pulpcore/app/models/fields.py @@ -79,7 +79,6 @@ def pre_save(self, model_instance, add): "prior to Artifact creation." ) ) - move = file._committed and file.name != artifact_storage_path if move: if not already_in_place: diff --git a/pulpcore/app/models/importer.py b/pulpcore/app/models/importer.py index 532fbada2e..4fa7f7ec04 100644 --- a/pulpcore/app/models/importer.py +++ b/pulpcore/app/models/importer.py @@ -5,6 +5,8 @@ BaseModel, MasterModel, ) +from pulpcore.app.util import get_domain_pk + from .repository import Repository @@ -23,6 +25,7 @@ class Import(BaseModel): params = models.JSONField(null=True) task = models.ForeignKey("Task", on_delete=models.PROTECT) importer = models.ForeignKey("Importer", on_delete=models.CASCADE) + pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT) class Importer(MasterModel): @@ -35,7 +38,11 @@ class Importer(MasterModel): name (models.TextField): The importer unique name. """ - name = models.TextField(db_index=True, unique=True) + name = models.TextField() + pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT) + + class Meta: + unique_together = ("name", "pulp_domain") class PulpImporter(Importer): diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py index 8b9b1d83ed..301d98695e 100644 --- a/pulpcore/app/models/repository.py +++ b/pulpcore/app/models/repository.py @@ -1105,6 +1105,7 @@ def remove_content(self, content): if not content or not content.count(): return + assert ( not Content.objects.filter(pk__in=content) .exclude(pulp_domain_id=get_domain_pk()) diff --git a/pulpcore/app/serializers/exporter.py b/pulpcore/app/serializers/exporter.py index aa071a7982..104b6d695b 100644 --- a/pulpcore/app/serializers/exporter.py +++ b/pulpcore/app/serializers/exporter.py @@ -3,12 +3,12 @@ import re from rest_framework import serializers -from rest_framework.validators import UniqueValidator from pulpcore.app import models, settings from pulpcore.app.serializers import ( DetailIdentityField, DetailRelatedField, + DomainUniqueValidator, ExportIdentityField, ExportRelatedField, ModelSerializer, @@ -37,8 +37,8 @@ class ExporterSerializer(ModelSerializer): pulp_href = DetailIdentityField(view_name_pattern=r"exporter(-.*/.*)-detail") name = serializers.CharField( - help_text=_("Unique name of the file system exporter."), - validators=[UniqueValidator(queryset=models.Exporter.objects.all())], + help_text=_("Unique name of the exporter."), + validators=[DomainUniqueValidator(queryset=models.Exporter.objects.all())], ) @staticmethod @@ -308,7 +308,7 @@ def validate(self, data): raise serializers.ValidationError( _("publication or repository_version must either be supplied but not both.") ) - return data + return super().validate(data) class Meta: model = models.FilesystemExport diff --git a/pulpcore/app/serializers/importer.py b/pulpcore/app/serializers/importer.py index 742ad9ab17..e1a28ae60a 100644 --- a/pulpcore/app/serializers/importer.py +++ b/pulpcore/app/serializers/importer.py @@ -3,11 +3,11 @@ from django.core.exceptions import ObjectDoesNotExist from rest_framework import serializers -from rest_framework.validators import UniqueValidator from pulpcore.app import models, settings from pulpcore.app.serializers import ( DetailIdentityField, + DomainUniqueValidator, ImportIdentityField, ModelSerializer, RelatedField, @@ -22,7 +22,7 @@ class ImporterSerializer(ModelSerializer): pulp_href = DetailIdentityField(view_name_pattern=r"importer(-.*/.*)-detail") name = serializers.CharField( help_text=_("Unique name of the Importer."), - validators=[UniqueValidator(queryset=models.Importer.objects.all())], + validators=[DomainUniqueValidator(queryset=models.Importer.objects.all())], ) class Meta: diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py index dfe14a5336..3e9b70913d 100644 --- a/pulpcore/app/tasks/importer.py +++ b/pulpcore/app/tasks/importer.py @@ -35,7 +35,12 @@ ContentArtifactResource, RepositoryResource, ) -from pulpcore.app.util import compute_file_hash, Crc32Hasher +from pulpcore.app.util import ( + compute_file_hash, + Crc32Hasher, + get_domain, + get_domain_pk, +) from pulpcore.constants import TASK_STATES from pulpcore.tasking.tasks import dispatch @@ -435,7 +440,6 @@ def pulp_import(importer_pk, path, toc, create_repositories): create_repositories (bool): Indicates whether missing repositories should be automatically created or not. """ - if toc: path = toc fileobj = ChunkedFile(toc) @@ -443,13 +447,12 @@ def pulp_import(importer_pk, path, toc, create_repositories): fileobj.validate_chunks() else: fileobj = nullcontext() - log.info(_("Importing {}.").format(path)) current_task = Task.current() task_group = TaskGroup.current() importer = PulpImporter.objects.get(pk=importer_pk) the_import = PulpImport.objects.create( - importer=importer, task=current_task, params={"path": path} + importer=importer, task=current_task, params={"path": path}, pulp_domain=get_domain() ) CreatedResource.objects.create(content_object=the_import) @@ -491,12 +494,39 @@ def safe_extract(tar, path=".", members=None, *, numeric_owner=False): for ar_result in _import_file(os.path.join(temp_dir, ARTIFACT_FILE), ArtifactResource): for row in pb.iter(ar_result.rows): artifact = Artifact.objects.get(pk=row.object_id) - base_path = os.path.join("artifact", artifact.sha256[0:2], artifact.sha256[2:]) - src = os.path.join(temp_dir, base_path) - if not default_storage.exists(base_path): - with open(src, "rb") as f: - default_storage.save(base_path, f) + # If we are domain-enabled, then the destination is "to the current + # domain's artifact directory". Otherwise, it's just to /artifact/. + if settings.DOMAIN_ENABLED: + destination_path = os.path.join( + "artifact", + str(get_domain_pk()), + artifact.sha256[0:2], + artifact.sha256[2:], + ) + + else: + destination_path = os.path.join( + "artifact", artifact.sha256[0:2], artifact.sha256[2:] + ) + + # If *the upstream* was domain-enabled, the tarfile will have artifact/DOMAIN/ + # in its path, and the Artifact will have artifact/current-domain-id in its + # "file" attribute. We need to copy from artifact/DOMAIN/ in the tarfile. + domain_path = os.path.join( + "artifact", "DOMAIN", artifact.sha256[0:2], artifact.sha256[2:] + ) + if os.path.exists(os.path.join(temp_dir, domain_path)): + tar_path = domain_path + else: + tar_path = os.path.join( + "artifact", artifact.sha256[0:2], artifact.sha256[2:] + ) + src_in_tar = os.path.join(temp_dir, tar_path) + + if not default_storage.exists(destination_path): + with open(src_in_tar, "rb") as f: + default_storage.save(destination_path, f) # Now import repositories, in parallel. @@ -529,7 +559,9 @@ def safe_extract(tar, path=".", members=None, *, numeric_owner=False): worker_rsrc = f"import-worker-{index % import_workers}" exclusive_resources = [worker_rsrc] try: - dest_repo = Repository.objects.get(name=dest_repo_name) + dest_repo = Repository.objects.get( + name=dest_repo_name, pulp_domain=get_domain() + ) except Repository.DoesNotExist: if create_repositories: dest_repo_pk = "" diff --git a/pulpcore/app/viewsets/exporter.py b/pulpcore/app/viewsets/exporter.py index 83888f0ee7..4f31de9474 100644 --- a/pulpcore/app/viewsets/exporter.py +++ b/pulpcore/app/viewsets/exporter.py @@ -1,6 +1,5 @@ -from django.conf import settings from drf_spectacular.utils import extend_schema -from rest_framework import mixins, exceptions +from rest_framework import mixins from pulpcore.app.models import ( Export, @@ -34,8 +33,6 @@ from pulpcore.plugin.tasking import dispatch from pulpcore.app.response import OperationPostponedResponse -from gettext import gettext as _ - class ExporterViewSet( NamedModelViewSet, @@ -120,8 +117,6 @@ def create(self, request, exporter_pk): """ Generates a Task to export the set of repositories assigned to a specific PulpExporter. """ - if settings.DOMAIN_ENABLED: - raise exceptions.ValidationError(_("Export not supported with Domains enabled.")) # Validate Exporter exporter = PulpExporter.objects.get(pk=exporter_pk).cast() ExporterSerializer.validate_path(exporter.path, check_is_dir=True) @@ -159,8 +154,6 @@ def create(self, request, exporter_pk): """ Generates a Task to export files to the filesystem. """ - if settings.DOMAIN_ENABLED: - raise exceptions.ValidationError(_("Export not supported with Domains enabled.")) # Validate Exporter exporter = FilesystemExporter.objects.get(pk=exporter_pk).cast() ExporterSerializer.validate_path(exporter.path, check_is_dir=True) diff --git a/pulpcore/app/viewsets/importer.py b/pulpcore/app/viewsets/importer.py index 80c2fe7aa6..d182a85f2d 100644 --- a/pulpcore/app/viewsets/importer.py +++ b/pulpcore/app/viewsets/importer.py @@ -1,7 +1,6 @@ -from django.conf import settings from django.http import Http404 from drf_spectacular.utils import extend_schema -from rest_framework import mixins, exceptions +from rest_framework import mixins from pulpcore.app.models import ( Import, @@ -25,8 +24,6 @@ from pulpcore.app.viewsets.base import NAME_FILTER_OPTIONS from pulpcore.tasking.tasks import dispatch -from gettext import gettext as _ - class ImporterViewSet( NamedModelViewSet, @@ -87,8 +84,6 @@ class PulpImportViewSet(ImportViewSet): ) def create(self, request, importer_pk): """Import a Pulp export into Pulp.""" - if settings.DOMAIN_ENABLED: - raise exceptions.ValidationError(_("Import not supported with Domains enabled.")) try: importer = PulpImporter.objects.get(pk=importer_pk) except PulpImporter.DoesNotExist: