added transcoding app from mitol and removed transcoding job code (#2421)

* added transcoding app from mitol and removed transcoding job code

* removed redundant code

* removed redundant settings

* refactored tests and updated transcoding url

* added dependency package `mitol-django-transcoding:2025.2.25`

* added missing settinsg for the transcode jobs test

* fixed value for the post transcoding action setting iin tests

* bump version for mitol-django-transcoding

* rebased, added support for local testing, and added missing settings

* removed local testing

* added latest transcoding app package

* removed redundant asserts

Author: umar8hassan

main/settings.py

@@ -69,6 +69,7 @@
     "mitol.common.settings.webpack",
     "mitol.mail.settings.email",
     "mitol.authentication.settings.touchstone",
+    "mitol.transcoding.settings.job",
 )
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -517,23 +518,6 @@
     description="Open catalog urls",
     required=False,
 )
-VIDEO_S3_TRANSCODE_PREFIX = get_string(
-    name="VIDEO_S3_TRANSCODE_PREFIX",
-    default="aws_mediaconvert_transcodes",
-    description=(
-        "Prefix to be used for S3 keys of files transcoded from AWS MediaConvert"
-    ),
-)
-VIDEO_S3_TRANSCODE_ENDPOINT = get_string(
-    name="VIDEO_S3_TRANSCODE_ENDPOINT",
-    default="aws_mediaconvert_transcodes",
-    description=("Endpoint to be used for AWS MediaConvert"),
-)
-VIDEO_TRANSCODE_QUEUE = get_string(
-    name="VIDEO_TRANSCODE_QUEUE",
-    default="Default",
-    description=("Name of MediaConvert queue to use for transcoding"),
-)
 YT_ACCESS_TOKEN = get_string(
     name="YT_ACCESS_TOKEN", default="", description="Youtube access token"
 )

main/urls.py

@@ -41,6 +41,7 @@
     path("", include("mitol.authentication.urls.saml")),
     path("", include("mitol.mail.urls")),
     path("", include("videos.urls")),
+    path("api/", include("mitol.transcoding.urls")),
 ]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

poetry.lock

@@ -11,7 +11,7 @@ package-mode = false
 python = "~3.12"
 
 beautifulsoup4 = "^4.12.2"
-boto3 = "1.37.4"
+boto3 = "1.37.11"
 celery = "^5.3.0"
 cryptography = "^44.0.0"
 dj-database-url = "2.3.0"
@@ -56,6 +56,7 @@ uwsgitop = "^0.12"
 yamale = "6.0.0"
 xmlsec = "1.3.13"
 posthog = "^3.7.0"
+mitol-django-transcoding = "2025.4.8"
 
 [tool.poetry.group.dev.dependencies]
 bpython = "^0.25"

pyproject.toml

@@ -1,18 +1,16 @@
 """APi functions for video processing"""
 
-import json
 import logging
 import os
 
-import boto3
 import botocore
 from django.conf import settings
+from mitol.transcoding.api import media_convert_job
 
 from content_sync.utils import move_s3_object
 from gdrive_sync.models import DriveFile
 from gdrive_sync.utils import fetch_content_file_size
 from main.s3_utils import get_boto3_resource
-from videos.apps import VideoApp
 from videos.constants import (
     DESTINATION_ARCHIVE,
     DESTINATION_YOUTUBE,
@@ -62,42 +60,11 @@ def prepare_video_download_file(video: Video):
 
 def create_media_convert_job(video: Video):
     """Create a MediaConvert job for a Video"""
-    source_prefix = settings.DRIVE_S3_UPLOAD_PREFIX
-    client = boto3.client(
-        "mediaconvert",
-        region_name=settings.AWS_REGION,
-        endpoint_url=settings.VIDEO_S3_TRANSCODE_ENDPOINT,
-    )
-    with open(  # noqa: PTH123
-        os.path.join(  # noqa: PTH118
-            settings.BASE_DIR, f"{VideoApp.name}/config/mediaconvert.json"
-        ),
-        encoding="utf-8",
-    ) as job_template:
-        job_dict = json.loads(job_template.read())
-        job_dict["UserMetadata"]["filter"] = settings.VIDEO_TRANSCODE_QUEUE
-        job_dict["Queue"] = (
-            f"arn:aws:mediaconvert:{settings.AWS_REGION}:{settings.AWS_ACCOUNT_ID}:queues/{settings.VIDEO_TRANSCODE_QUEUE}"
-        )
-        job_dict["Role"] = (
-            f"arn:aws:iam::{settings.AWS_ACCOUNT_ID}:role/{settings.AWS_ROLE_NAME}"
-        )
-        destination = os.path.splitext(  # noqa: PTH122
-            video.source_key.replace(
-                source_prefix,
-                settings.VIDEO_S3_TRANSCODE_PREFIX,
-            )
-        )[0]
-        job_dict["Settings"]["OutputGroups"][0]["OutputGroupSettings"][
-            "FileGroupSettings"
-        ]["Destination"] = f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{destination}"
-        job_dict["Settings"]["Inputs"][0]["FileInput"] = (
-            f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{video.source_key}"
-        )
-        job = client.create_job(**job_dict)
-        VideoJob.objects.get_or_create(video=video, job_id=job["Job"]["Id"])
-        video.status = VideoStatus.TRANSCODING
-        video.save()
+
+    job = media_convert_job(video.source_key)
+    VideoJob.objects.get_or_create(video=video, job_id=job["Job"]["Id"])
+    video.status = VideoStatus.TRANSCODING
+    video.save()
 
 
 def process_video_outputs(video: Video, output_group_details: dict):
@@ -124,8 +91,10 @@ def process_video_outputs(video: Video, output_group_details: dict):
     prepare_video_download_file(video)
 
 
-def update_video_job(video_job: VideoJob, results: dict):
+def update_video_job(results: dict):
     """Update a VideoJob and associated Video, VideoFiles based on MediaConvert results"""  # noqa: E501
+
+    video_job = VideoJob.objects.get(job_id=results.get("jobId"))
     video_job.job_output = results
     status = results.get("status")
     video = video_job.video

videos/api.py

@@ -32,7 +32,7 @@ def test_create_media_convert_job(settings, mocker):
     """create_media_convert_job should send a request to MediaConvert, create a VideoJob object"""
     queue_name = "test_queue"
     settings.VIDEO_TRANSCODE_QUEUE = queue_name
-    mock_boto = mocker.patch("videos.api.boto3")
+    mock_boto = mocker.patch("mitol.transcoding.api.boto3")
     job_id = "abcd123-gh564"
     mock_boto.client.return_value.create_job.return_value = {"Job": {"Id": job_id}}
     video = VideoFactory.create()
@@ -49,16 +49,9 @@ def test_create_media_convert_job(settings, mocker):
     destination = call_kwargs["Settings"]["OutputGroups"][0]["OutputGroupSettings"][
         "FileGroupSettings"
     ]["Destination"]
-    assert destination.startswith(
-        f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{settings.VIDEO_S3_TRANSCODE_PREFIX}"
-    )
     assert destination.endswith(
         path.splitext(video.source_key.split("/")[-1])[0]  # noqa: PTH122
     )
-    assert (
-        call_kwargs["Settings"]["Inputs"][0]["FileInput"]
-        == f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{video.source_key}"
-    )
     assert VideoJob.objects.filter(job_id=job_id, video=video).count() == 1
     video.refresh_from_db()
     assert video.status == VideoStatus.TRANSCODING
@@ -132,12 +125,14 @@ def test_update_video_job_success(mocker, raises_exception):
     )
     mock_log = mocker.patch("videos.api.log.exception")
     video_job = VideoJobFactory.create(status=VideoJobStatus.CREATED)
+    mock_job = mocker.patch("videos.api.VideoJob.objects.get")
+    mock_job.return_value = video_job
     with open(  # noqa: PTH123
         f"{TEST_VIDEOS_WEBHOOK_PATH}/cloudwatch_sns_complete.json",
         encoding="utf-8",
     ) as infile:
         data = json.loads(infile.read())["detail"]
-    update_video_job(video_job, data)
+    update_video_job(data)
     mock_process_outputs.assert_called_once()
     video_job.refresh_from_db()
     assert video_job.job_output == data
@@ -149,11 +144,13 @@ def test_update_video_job_error(mocker):
     """The video job should be updated as expected if the transcode job failed"""
     mock_log = mocker.patch("videos.api.log.error")
     video_job = VideoJobFactory.create()
+    mock_job = mocker.patch("videos.api.VideoJob.objects.get")
+    mock_job.return_value = video_job
     with open(  # noqa: PTH123
         f"{TEST_VIDEOS_WEBHOOK_PATH}/cloudwatch_sns_error.json", encoding="utf-8"
     ) as infile:
         data = json.loads(infile.read())["detail"]
-    update_video_job(video_job, data)
+    update_video_job(data)
     video_job.refresh_from_db()
     assert video_job.job_output == data
     assert video_job.error_code == str(data.get("errorCode"))