refactor: remove import_audio_files() and related code

ietf-tools · rjsparks · Mar 10, 2023 · Mar 1, 2023 · Mar 1, 2023 · Mar 1, 2023
commit 420dc414e7919dbd6cdb1e1a18e7a22640e71533
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
@@ -8,7 +8,6 @@
 import sys
 
 from importlib import import_module
-from mock import patch
 from pathlib import Path
 
 from django.apps import apps
@@ -27,7 +26,6 @@
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory
 from ietf.group.factories import RoleFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory
-from ietf.meeting.test_data import make_meeting_test_data
 from ietf.meeting.models import Session
 from ietf.person.factories import PersonFactory, random_faker
 from ietf.person.models import User
@@ -46,20 +44,6 @@
 class CustomApiTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
 
-    # Using mock to patch the import functions in ietf.meeting.views, where
-    # api_import_recordings() are using them:
-    @patch('ietf.meeting.views.import_audio_files')
-    def test_notify_meeting_import_audio_files(self, mock_import_audio):
-        meeting = make_meeting_test_data()
-        client = Client(Accept='application/json')
-        # try invalid method GET
-        url = urlreverse('ietf.meeting.views.api_import_recordings', kwargs={'number':meeting.number})
-        r = client.get(url)
-        self.assertEqual(r.status_code, 405)
-        # try valid method POST
-        r = client.post(url)
-        self.assertEqual(r.status_code, 201)
-
     def test_api_help_page(self):
         url = urlreverse('ietf.api.views.api_help')
         r = self.client.get(url)

diff --git a/ietf/api/urls.py b/ietf/api/urls.py
@@ -32,8 +32,6 @@
     url(r'^meeting/(?P<num>[A-Za-z0-9._+-]+)/agenda-data$', meeting_views.api_get_agenda_data),
     # Meeting session materials
     url(r'^meeting/session/(?P<session_id>[A-Za-z0-9._+-]+)/materials$', meeting_views.api_get_session_materials),
-    # Let Meetecho trigger recording imports
-    url(r'^notify/meeting/import_recordings/(?P<number>[a-z0-9-]+)/?$', meeting_views.api_import_recordings),
     # Let MeetEcho upload bluesheets
     url(r'^notify/meeting/bluesheet/?$', meeting_views.api_upload_bluesheet),
     # Let MeetEcho tell us about session attendees

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
@@ -25,7 +25,6 @@
 from ietf.group.utils import can_manage_materials
 from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName
 from ietf.person.models import Person
-from ietf.secr.proceedings.proc_utils import import_audio_files
 from ietf.utils.html import sanitize_document
 from ietf.utils.log import log
 from ietf.utils.timezone import date_today
@@ -180,7 +179,6 @@ def finalize(meeting):
                 sp.rev = '00'
             sp.save()
 
-    import_audio_files(meeting)
     create_proceedings_templates(meeting)
     meeting.proceedings_final = True
     meeting.save()

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
@@ -84,8 +84,7 @@
 from ietf.meeting.utils import new_doc_for_session, write_doc_for_session
 from ietf.message.utils import infer_message
 from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
-from ietf.secr.proceedings.proc_utils import (get_activity_stats, post_process, import_audio_files,
-    create_recording)
+from ietf.secr.proceedings.proc_utils import get_activity_stats, post_process, create_recording
 from ietf.utils import markdown
 from ietf.utils.decorators import require_api_key
 from ietf.utils.hedgedoc import Note, NoteError
@@ -3798,16 +3797,6 @@ class OldUploadRedirect(RedirectView):
     def get_redirect_url(self, **kwargs):
         return reverse_lazy('ietf.meeting.views.session_details',kwargs=self.kwargs)
 
-@csrf_exempt
-def api_import_recordings(request, number):
-    '''REST API to check for recording files and import'''
-    if request.method == 'POST':
-        meeting = get_meeting(number)
-        import_audio_files(meeting)
-        return HttpResponse(status=201)
-    else:
-        return HttpResponse(status=405)
-
 @require_api_key
 @role_required('Recording Manager')
 @csrf_exempt

diff --git a/ietf/secr/proceedings/proc_utils.py b/ietf/secr/proceedings/proc_utils.py
@@ -10,26 +10,20 @@
 import datetime
 import os
 import pytz
-import re
 import subprocess
 from urllib.parse import urlencode
 
 import debug        # pyflakes:ignore
 
 from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist
 
 from ietf.doc.models import Document, DocAlias, DocEvent, NewRevisionDocEvent, State
 from ietf.group.models import Group
-from ietf.meeting.models import Meeting, SessionPresentation, TimeSlot, SchedTimeSessAssignment, Session
+from ietf.meeting.models import Meeting, SessionPresentation, SchedTimeSessAssignment
 from ietf.person.models import Person
 from ietf.utils.log import log
-from ietf.utils.mail import send_mail
 from ietf.utils.timezone import make_aware
 
-AUDIO_FILE_RE = re.compile(r'ietf(?P<number>[\d]+)-(?P<room>.*)-(?P<time>[\d]{8}-[\d]{4})')
-VIDEO_TITLE_RE = re.compile(r'IETF(?P<number>[\d]+)-(?P<name>.*)-(?P<date>\d{8})-(?P<time>\d{4})')
-
 
 def _get_session(number,name,date,time):
     '''Lookup session using data from video title'''
@@ -57,86 +51,6 @@ def _get_urls_from_json(doc):
         urls.append(dict(title=title, url=url))
     return urls
 
-def import_audio_files(meeting):
-    '''
-    Checks for audio files and creates corresponding materials (docs) for the Session
-    Expects audio files in the format ietf[meeting num]-[room]-YYYMMDD-HHMM.*,
-
-    Example: ietf90-salonb-20140721-1710.mp3
-    '''
-    unmatched_files = []
-    path = os.path.join(settings.MEETING_RECORDINGS_DIR, meeting.type.slug + meeting.number)
-    if not os.path.exists(path):
-        return None
-    for filename in os.listdir(path):
-        timeslot = get_timeslot_for_filename(filename)
-        if timeslot:
-            sessions = Session.objects.with_current_status().filter(
-                timeslotassignments__schedule=timeslot.meeting.schedule_id,
-            ).filter(
-                current_status='sched',
-            ).order_by('timeslotassignments__timeslot__time')
-            if not sessions:
-                continue
-            url = settings.IETF_AUDIO_URL + 'ietf{}/{}'.format(meeting.number, filename)
-            doc = get_or_create_recording_document(url, sessions[0])
-            attach_recording(doc, sessions)
-        else:
-            # use for reconciliation email
-            unmatched_files.append(filename)
-
-    if unmatched_files:
-        send_audio_import_warning(unmatched_files)
-
-def get_timeslot_for_filename(filename):
-    '''Returns a timeslot matching the filename given.
-    NOTE: currently only works with ietfNN prefix (regular meetings)
-    '''
-    from ietf.meeting.utils import add_event_info_to_session_qs
-
-    basename, _ = os.path.splitext(filename)
-    match = AUDIO_FILE_RE.match(basename)
-    if match:
-        try:
-            meeting = Meeting.objects.get(number=match.groupdict()['number'])
-            room_mapping = {normalize_room_name(room.name): room.name for room in meeting.room_set.all()}
-            time = make_aware(datetime.datetime.strptime(match.groupdict()['time'],'%Y%m%d-%H%M'), meeting.tz())
-            slots = TimeSlot.objects.filter(
-                meeting=meeting,
-                location__name=room_mapping[match.groupdict()['room']],
-                time=time,
-                sessionassignments__schedule__in=[meeting.schedule, meeting.schedule.base if meeting.schedule else None],
-            ).distinct()
-            uncancelled_slots = [t for t in slots if not add_event_info_to_session_qs(t.sessions.all()).filter(current_status='canceled').exists()]
-            return uncancelled_slots[0]
-        except (ObjectDoesNotExist, KeyError, IndexError):
-            return None
-
-def attach_recording(doc, sessions):
-    '''Associate recording document with sessions'''
-    for session in sessions:
-        if doc not in session.materials.all():
-            # add document to session
-            presentation = SessionPresentation.objects.create(
-                session=session,
-                document=doc,
-                rev=doc.rev)
-            session.sessionpresentation_set.add(presentation)
-            if not doc.docalias.filter(name__startswith='recording-{}-{}'.format(session.meeting.number,session.group.acronym)):
-                sequence = get_next_sequence(session.group,session.meeting,'recording')
-                name = 'recording-{}-{}-{}'.format(session.meeting.number,session.group.acronym,sequence)
-                DocAlias.objects.create(name=name).docs.add(doc)
-
-def normalize_room_name(name):
-    '''Returns room name converted to be used as portion of filename'''
-    return name.lower().replace(' ','').replace('/','_')
-
-def get_or_create_recording_document(url,session):
-    try:
-        return Document.objects.get(external_url=url)
-    except ObjectDoesNotExist:
-        return create_recording(session,url)
-
 def create_recording(session, url, title=None, user=None):
     '''
     Creates the Document type=recording, setting external_url and creating
@@ -185,19 +99,6 @@ def get_next_sequence(group,meeting,type):
     sequence = int(aliases.last().name.split('-')[-1]) + 1
     return sequence
 
-def send_audio_import_warning(unmatched_files):
-    '''Send email to interested parties that some audio files weren't matched to timeslots'''
-    send_mail(request = None,
-              to       = settings.AUDIO_IMPORT_EMAIL,
-              frm      = "IETF Secretariat <ietf-secretariat@ietf.org>",
-              subject  = "Audio file import warning",
-              template = "proceedings/audio_import_warning.txt",
-              context  = dict(unmatched_files=unmatched_files),
-              extra    = {})
-
-# -------------------------------------------------
-# End Recording Functions
-# -------------------------------------------------
 
 def get_activity_stats(sdate, edate):
     '''

diff --git a/ietf/secr/proceedings/tests.py b/ietf/secr/proceedings/tests.py
@@ -9,23 +9,14 @@
 
 from django.conf import settings
 from django.urls import reverse
-
-from ietf.doc.models import Document
 from ietf.group.factories import RoleFactory
-from ietf.meeting.models import SchedTimeSessAssignment, SchedulingEvent
 from ietf.meeting.factories import MeetingFactory, SessionFactory
-from ietf.person.models import Person
-from ietf.name.models import SessionStatusName
 from ietf.utils.test_utils import TestCase
-from ietf.utils.mail import outbox
 
-from ietf.secr.proceedings.proc_utils import (import_audio_files,
-    get_timeslot_for_filename, normalize_room_name, send_audio_import_warning,
-    get_or_create_recording_document, create_recording, get_next_sequence,
-    _get_session, _get_urls_from_json)
 
+from ietf.secr.proceedings.proc_utils import (create_recording, 
+    get_next_sequence, _get_session, _get_urls_from_json)
 
-SECR_USER='secretary'
 
 class ProceedingsTestCase(TestCase):
     def test_main(self):
@@ -92,81 +83,6 @@ def test_post(self):
         response = self.client.post(url,dict(external_url=external_url),follow=True)
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, external_url)
-
-    def test_import_audio_files(self):
-        session = SessionFactory(status_id='sched',meeting__type_id='ietf')
-        meeting = session.meeting
-        timeslot = session.official_timeslotassignment().timeslot
-        self.create_audio_file_for_timeslot(timeslot)
-        import_audio_files(meeting)
-        self.assertEqual(session.materials.filter(type='recording').count(),1)
-
-    def create_audio_file_for_timeslot(self, timeslot):
-        filename = self.get_filename_for_timeslot(timeslot)
-        path = os.path.join(settings.MEETING_RECORDINGS_DIR,'ietf' + timeslot.meeting.number,filename)
-        if not os.path.exists(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-        with io.open(path, "w") as f:
-            f.write('dummy')
-
-    def get_filename_for_timeslot(self, timeslot):
-        '''Returns the filename of a session recording given timeslot'''
-        return "{prefix}-{room}-{date}.mp3".format(
-            prefix=timeslot.meeting.type.slug + timeslot.meeting.number,
-            room=normalize_room_name(timeslot.location.name),
-            date=timeslot.local_start_time().strftime('%Y%m%d-%H%M'))
-
-    def test_import_audio_files_shared_timeslot(self):
-        meeting = MeetingFactory(type_id='ietf',number='72')
-        mars_session = SessionFactory(meeting=meeting,status_id='sched',group__acronym='mars')
-        ames_session = SessionFactory(meeting=meeting,status_id='sched',group__acronym='ames')
-        scheduled = SessionStatusName.objects.get(slug='sched')
-        SchedulingEvent.objects.create(
-            session=mars_session,
-            status=scheduled,
-            by=Person.objects.get(name='(System)')
-        )
-        SchedulingEvent.objects.create(
-            session=ames_session,
-            status=scheduled,
-            by=Person.objects.get(name='(System)')
-        )
-        timeslot = mars_session.official_timeslotassignment().timeslot
-        SchedTimeSessAssignment.objects.create(timeslot=timeslot,session=ames_session,schedule=meeting.schedule)
-        self.create_audio_file_for_timeslot(timeslot)
-        import_audio_files(meeting)
-        doc = mars_session.materials.filter(type='recording').first()
-        self.assertTrue(doc in ames_session.materials.all())
-        self.assertTrue(doc.docalias.filter(name='recording-72-mars-1'))
-        self.assertTrue(doc.docalias.filter(name='recording-72-ames-1'))
-
-    def test_normalize_room_name(self):
-        self.assertEqual(normalize_room_name('Test Room'),'testroom')
-        self.assertEqual(normalize_room_name('Rome/Venice'), 'rome_venice')
-
-    def test_get_timeslot_for_filename(self):
-        session = SessionFactory(meeting__type_id='ietf')
-        timeslot = session.timeslotassignments.first().timeslot
-        name = self.get_filename_for_timeslot(timeslot)
-        self.assertEqual(get_timeslot_for_filename(name),timeslot)
-
-    def test_get_or_create_recording_document(self):
-        session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars')
-
-        # test create
-        filename = 'ietf42-testroom-20000101-0800.mp3'
-        docs_before = Document.objects.filter(type='recording').count()
-        doc = get_or_create_recording_document(filename,session)
-        docs_after = Document.objects.filter(type='recording').count()
-        self.assertEqual(docs_after,docs_before + 1)
-        self.assertTrue(doc.external_url.endswith(filename))
-
-        # test get
-        docs_before = docs_after
-        doc2 = get_or_create_recording_document(filename,session)
-        docs_after = Document.objects.filter(type='recording').count()
-        self.assertEqual(docs_after,docs_before)
-        self.assertEqual(doc,doc2)
 
     def test_create_recording(self):
         session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars')
@@ -184,9 +100,3 @@ def test_get_next_sequence(self):
         group = session.group
         sequence = get_next_sequence(group,meeting,'recording')
         self.assertEqual(sequence,1)
-
-    def test_send_audio_import_warning(self):
-        length_before = len(outbox)
-        send_audio_import_warning(['recording-43-badroom-20000101-0800.mp3'])
-        self.assertEqual(len(outbox), length_before + 1)
-        self.assertTrue('Audio file import' in outbox[-1]['Subject'])
diff --git a/ietf/secr/templates/proceedings/audio_import_warning.txt b/ietf/secr/templates/proceedings/audio_import_warning.txt