Merge pull request #1142 from ATIX-AG/cleanup_tmp_files

Fixed cleanup of temporary files from tasks

CHANGES/1141.bugfix

@@ -0,0 +1 @@
+Fixed an issue where the signing service did not properly clean up temporary files after completion.

docs/user/guides/signing_service.md

@@ -36,7 +36,7 @@ The following example signing service script is used as part of the `pulp_deb` t
 set -e
 
 RELEASE_FILE="$(/usr/bin/readlink -f $1)"
-OUTPUT_DIR="$(/usr/bin/mktemp -d)"
+OUTPUT_DIR="${PULP_TEMP_WORKING_DIR}"
 DETACHED_SIGNATURE_PATH="${OUTPUT_DIR}/Release.gpg"
 INLINE_SIGNATURE_PATH="${OUTPUT_DIR}/InRelease"
 GPG_KEY_ID="Pulp QE"

pulp_deb/app/models/signing_service.py

@@ -34,11 +34,12 @@ def validate(self):
         """
         with tempfile.TemporaryDirectory() as temp_directory_name:
             test_release_path = os.path.join(temp_directory_name, "Release")
+            temp_env = {"PULP_TEMP_WORKING_DIR": temp_directory_name}
             with open(test_release_path, "wb") as test_file:
                 test_data = b"arbitrary data"
                 test_file.write(test_data)
                 test_file.flush()
-                return_value = self.sign(test_release_path)
+                return_value = self.sign(test_release_path, env_vars=temp_env)
 
                 signatures = return_value.get("signatures")
 

pulp_deb/app/tasks/publishing.py

@@ -1,6 +1,8 @@
 import asyncio
 import os
 import shutil
+import random
+import string
 from contextlib import suppress
 from pathlib import Path
 
@@ -112,7 +114,7 @@ def publish(
             structured=structured,
         )
     )
-    with tempfile.TemporaryDirectory("."):
+    with tempfile.TemporaryDirectory(".") as temp_dir:
         with AptPublication.create(repo_version, pass_through=False) as publication:
             publication.simple = simple
             publication.structured = structured
@@ -144,6 +146,7 @@ def publish(
                     release=release,
                     components=[component],
                     architectures=architectures,
+                    temp_dir=temp_dir,
                     signing_service=repository.signing_service,
                 )
 
@@ -239,6 +242,7 @@ def publish(
                         components=components,
                         architectures=architectures,
                         release=release,
+                        temp_dir=temp_dir,
                         signing_service=signing_service,
                     )
 
@@ -465,9 +469,11 @@ def __init__(
         components,
         architectures,
         release,
+        temp_dir,
         signing_service=None,
     ):
         self.publication = publication
+        self.temp_env = {"PULP_TEMP_WORKING_DIR": _create_random_directory(temp_dir)}
         self.distribution = distribution = release.distribution
         self.dists_subfolder = distribution.strip("/") if distribution != "/" else "flat-repo"
         if distribution[-1] == "/":
@@ -548,7 +554,9 @@ def save_unsigned_metadata(self):
     async def sign_metadata(self):
         self.signed = {"signatures": {}}
         if self.signing_service:
-            self.signed = await self.signing_service.asign(self.release_path)
+            self.signed = await self.signing_service.asign(
+                self.release_path, env_vars=self.temp_env
+            )
 
     def save_signed_metadata(self):
         for signature_file in self.signed["signatures"].values():
@@ -586,3 +594,10 @@ def _batch_fetch_artifacts(packages):
     remote_artifact_dict = {artifact.sha256: artifact for artifact in remote_artifacts}
 
     return artifact_dict, remote_artifact_dict
+
+
+def _create_random_directory(path):
+    dir_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
+    dir_path = path + "/" + dir_name
+    os.makedirs(dir_path, exist_ok=True)
+    return dir_path

pulp_deb/app/tasks/synchronizing.py

@@ -502,16 +502,22 @@ async def run(self):
                         # No main_artifact found, uncompress one
                         relative_dir = os.path.dirname(d_content.content.relative_path)
                         filename = _uncompress_artifact(d_content.d_artifacts, relative_dir)
-                        da = DeclarativeArtifact(
-                            artifact=Artifact.init_and_validate(
-                                filename, expected_digests={"sha256": content.sha256}
-                            ),
-                            url=filename,
-                            relative_path=content.relative_path,
-                            remote=d_content.d_artifacts[0].remote,
-                        )
-                        d_content.d_artifacts.append(da)
-                        await _save_artifact_blocking(da)
+
+                        try:
+                            da = DeclarativeArtifact(
+                                artifact=Artifact.init_and_validate(
+                                    filename, expected_digests={"sha256": content.sha256}
+                                ),
+                                url=filename,
+                                relative_path=content.relative_path,
+                                remote=d_content.d_artifacts[0].remote,
+                            )
+                            d_content.d_artifacts.append(da)
+                            await _save_artifact_blocking(da)
+                        finally:
+                            # Ensure the uncompressed file is deleted after usage
+                            if os.path.exists(filename):
+                                os.remove(filename)
                     content.artifact_set_sha256 = _get_artifact_set_sha256(
                         d_content, PackageIndex.SUPPORTED_ARTIFACTS
                     )

pulp_deb/tests/functional/conftest.py

@@ -486,10 +486,11 @@ def deb_signing_script_path(
     return signing_script_filename
 
 
-@pytest.fixture(scope="class")
+@pytest.fixture(scope="session")
 def deb_signing_service_factory(
     deb_signing_script_path,
     signing_gpg_metadata,
+    signing_gpg_homedir_path,
     pulpcore_bindings,
 ):
     """A fixture for the debian signing service."""
@@ -518,10 +519,17 @@ def deb_signing_service_factory(
     yield signing_service
 
     cmd = (
-        "from pulpcore.app.models import SigningService;"
-        f"SigningService.objects.filter(name='{service_name}').delete()"
+        "pulpcore-manager",
+        "remove-signing-service",
+        service_name,
+        "--class",
+        "deb:AptReleaseSigningService",
+    )
+    process = subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
     )
-    process = subprocess.run(["pulpcore-manager", "shell", "-c", cmd], capture_output=True)
     assert process.returncode == 0
 
 

pulp_deb/tests/functional/constants.py

@@ -374,7 +374,7 @@ def _clean_dict(d):
 
 export GNUPGHOME="HOMEDIRHERE"
 RELEASE_FILE="$(/usr/bin/readlink -f $1)"
-OUTPUT_DIR="$(/usr/bin/mktemp -d)"
+OUTPUT_DIR="${PULP_TEMP_WORKING_DIR}"
 DETACHED_SIGNATURE_PATH="${OUTPUT_DIR}/Release.gpg"
 INLINE_SIGNATURE_PATH="${OUTPUT_DIR}/InRelease"
 GPG_KEY_ID="GPGKEYIDHERE"