refactor: Remove secr proceedings (#5256)

* refactor: remove import_audio_files() and related code * refactor: move functions from proc_utils to meeting/utils * refactor: remove secr/proceedings
ietf-tools · Mar 10, 2023 · b654b49 · b654b49
1 parent 61504b1
commit b654b49
Show file tree

Hide file tree

Showing 34 changed files with 178 additions and 1,620 deletions.
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
@@ -8,7 +8,6 @@
 import sys
 
 from importlib import import_module
-from mock import patch
 from pathlib import Path
 
 from django.apps import apps
@@ -27,7 +26,6 @@
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory
 from ietf.group.factories import RoleFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory
-from ietf.meeting.test_data import make_meeting_test_data
 from ietf.meeting.models import Session
 from ietf.person.factories import PersonFactory, random_faker
 from ietf.person.models import User
@@ -46,20 +44,6 @@
 class CustomApiTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
 
-    # Using mock to patch the import functions in ietf.meeting.views, where
-    # api_import_recordings() are using them:
-    @patch('ietf.meeting.views.import_audio_files')
-    def test_notify_meeting_import_audio_files(self, mock_import_audio):
-        meeting = make_meeting_test_data()
-        client = Client(Accept='application/json')
-        # try invalid method GET
-        url = urlreverse('ietf.meeting.views.api_import_recordings', kwargs={'number':meeting.number})
-        r = client.get(url)
-        self.assertEqual(r.status_code, 405)
-        # try valid method POST
-        r = client.post(url)
-        self.assertEqual(r.status_code, 201)
-
     def test_api_help_page(self):
         url = urlreverse('ietf.api.views.api_help')
         r = self.client.get(url)

diff --git a/ietf/api/urls.py b/ietf/api/urls.py
@@ -32,8 +32,6 @@
     url(r'^meeting/(?P<num>[A-Za-z0-9._+-]+)/agenda-data$', meeting_views.api_get_agenda_data),
     # Meeting session materials
     url(r'^meeting/session/(?P<session_id>[A-Za-z0-9._+-]+)/materials$', meeting_views.api_get_session_materials),
-    # Let Meetecho trigger recording imports
-    url(r'^notify/meeting/import_recordings/(?P<number>[a-z0-9-]+)/?$', meeting_views.api_import_recordings),
     # Let MeetEcho upload bluesheets
     url(r'^notify/meeting/bluesheet/?$', meeting_views.api_upload_bluesheet),
     # Let MeetEcho tell us about session attendees

diff --git a/ietf/doc/management/commands/fix_105_slides.py b/ietf/doc/management/commands/fix_105_slides.py
@@ -12,7 +12,7 @@
 from ietf.meeting.models import Meeting, SessionPresentation
 from ietf.person.models import Person
 
-from ietf.secr.proceedings.proc_utils import is_powerpoint, post_process
+from ietf.meeting.utils import is_powerpoint, post_process
 
 class Command(BaseCommand):
     help = ('Fix uploaded_filename and generate pdf from pptx')

diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
@@ -63,7 +63,7 @@
 from ietf.iesg.utils import telechat_page_count
 from ietf.ietfauth.utils import has_role, role_required, user_is_person
 from ietf.person.models import Person
-from ietf.secr.proceedings.proc_utils import get_activity_stats
+from ietf.meeting.utils import get_activity_stats
 from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date
 from ietf.utils.timezone import date_today, datetime_from_date
 

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
@@ -47,6 +47,7 @@
 from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data
 from ietf.meeting.utils import finalize, condition_slide_order
 from ietf.meeting.utils import add_event_info_to_session_qs
+from ietf.meeting.utils import create_recording, get_next_sequence
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName
 from ietf.utils.decorators import skip_coverage
@@ -8095,3 +8096,20 @@ def test_rename_proceedings_material(self):
         pm = meeting.proceedings_materials.get(pk=pm.pk)
         self.assertEqual(str(pm), 'This Is Not the Default Name')
         self.assertEqual(pm.document.rev, orig_rev, 'Renaming should not change document revision')
+
+    def test_create_recording(self):
+        session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars')
+        filename = 'ietf42-testroomt-20000101-0800.mp3'
+        url = settings.IETF_AUDIO_URL + 'ietf{}/{}'.format(session.meeting.number, filename)
+        doc = create_recording(session, url)
+        self.assertEqual(doc.name,'recording-72-mars-1')
+        self.assertEqual(doc.group,session.group)
+        self.assertEqual(doc.external_url,url)
+        self.assertTrue(doc in session.materials.all())
+
+    def test_get_next_sequence(self):
+        session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars')
+        meeting = session.meeting
+        group = session.group
+        sequence = get_next_sequence(group,meeting,'recording')
+        self.assertEqual(sequence,1)
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 import datetime
 import itertools
+import os
 import pytz
 import requests
 import subprocess
@@ -19,13 +20,14 @@
 import debug                            # pyflakes:ignore
 
 from ietf.dbtemplate.models import DBTemplate
-from ietf.meeting.models import Session, SchedulingEvent, TimeSlot, Constraint, SchedTimeSessAssignment
+from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot,
+    Constraint, SchedTimeSessAssignment, SessionPresentation)
 from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent
+from ietf.doc.models import DocEvent
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_materials
 from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName
 from ietf.person.models import Person
-from ietf.secr.proceedings.proc_utils import import_audio_files
 from ietf.utils.html import sanitize_document
 from ietf.utils.log import log
 from ietf.utils.timezone import date_today
@@ -180,7 +182,6 @@ def finalize(meeting):
                 sp.rev = '00'
             sp.save()
 
-    import_audio_files(meeting)
     create_proceedings_templates(meeting)
     meeting.proceedings_final = True
     meeting.save()
@@ -756,3 +757,156 @@ def write_doc_for_session(session, type_id, filename, contents):
     with open(path / filename, "wb") as file:
         file.write(contents.encode('utf-8'))
     return
+
+def create_recording(session, url, title=None, user=None):
+    '''
+    Creates the Document type=recording, setting external_url and creating
+    NewRevisionDocEvent
+    '''
+    sequence = get_next_sequence(session.group,session.meeting,'recording')
+    name = 'recording-{}-{}-{}'.format(session.meeting.number,session.group.acronym,sequence)
+    time = session.official_timeslotassignment().timeslot.time.strftime('%Y-%m-%d %H:%M')
+    if not title:
+        if url.endswith('mp3'):
+            title = 'Audio recording for {}'.format(time)
+        else:
+            title = 'Video recording for {}'.format(time)
+
+    doc = Document.objects.create(name=name,
+                                  title=title,
+                                  external_url=url,
+                                  group=session.group,
+                                  rev='00',
+                                  type_id='recording')
+    doc.set_state(State.objects.get(type='recording', slug='active'))
+
+    DocAlias.objects.create(name=doc.name).docs.add(doc)
+
+    # create DocEvent
+    NewRevisionDocEvent.objects.create(type='new_revision',
+                                       by=user or Person.objects.get(name='(System)'),
+                                       doc=doc,
+                                       rev=doc.rev,
+                                       desc='New revision available',
+                                       time=doc.time)
+    pres = SessionPresentation.objects.create(session=session,document=doc,rev=doc.rev)
+    session.sessionpresentation_set.add(pres)
+
+    return doc
+
+def get_next_sequence(group, meeting, type):
+    '''
+    Returns the next sequence number to use for a document of type = type.
+    Takes a group=Group object, meeting=Meeting object, type = string
+    '''
+    aliases = DocAlias.objects.filter(name__startswith='{}-{}-{}-'.format(type, meeting.number, group.acronym))
+    if not aliases:
+        return 1
+    aliases = aliases.order_by('name')
+    sequence = int(aliases.last().name.split('-')[-1]) + 1
+    return sequence
+
+def get_activity_stats(sdate, edate):
+    '''
+    This function takes a date range and produces a dictionary of statistics / objects for
+    use in an activity report.  Generally the end date will be the date of the last meeting
+    and the start date will be the date of the meeting before that.
+
+    Data between midnight UTC on the specified dates are included in the stats.
+    '''
+    sdatetime = pytz.utc.localize(datetime.datetime.combine(sdate, datetime.time()))
+    edatetime = pytz.utc.localize(datetime.datetime.combine(edate, datetime.time()))
+
+    data = {}
+    data['sdate'] = sdate
+    data['edate'] = edate
+
+    events = DocEvent.objects.filter(doc__type='draft', time__gte=sdatetime, time__lt=edatetime)
+
+    data['actions_count'] = events.filter(type='iesg_approved').count()
+    data['last_calls_count'] = events.filter(type='sent_last_call').count()
+    new_draft_events = events.filter(newrevisiondocevent__rev='00')
+    new_drafts = list(set([e.doc_id for e in new_draft_events]))
+    data['new_docs'] = list(set([e.doc for e in new_draft_events]))
+    data['new_drafts_count'] = len(new_drafts)
+    data['new_drafts_updated_count'] = events.filter(doc__id__in=new_drafts,newrevisiondocevent__rev='01').count()
+    data['new_drafts_updated_more_count'] = events.filter(doc__id__in=new_drafts,newrevisiondocevent__rev='02').count()
+
+    update_events = events.filter(type='new_revision').exclude(doc__id__in=new_drafts)
+    data['updated_drafts_count'] = len(set([e.doc_id for e in update_events]))
+
+    # Calculate Final Four Weeks stats (ffw)
+    ffwdate = edatetime - datetime.timedelta(days=28)
+    ffw_new_count = events.filter(time__gte=ffwdate, newrevisiondocevent__rev='00').count()
+    try:
+        ffw_new_percent = format(ffw_new_count / float(data['new_drafts_count']), '.0%')
+    except ZeroDivisionError:
+        ffw_new_percent = 0
+
+    data['ffw_new_count'] = ffw_new_count
+    data['ffw_new_percent'] = ffw_new_percent
+
+    ffw_update_events = events.filter(time__gte=ffwdate, type='new_revision').exclude(doc__id__in=new_drafts)
+    ffw_update_count = len(set([e.doc_id for e in ffw_update_events]))
+    try:
+        ffw_update_percent = format(ffw_update_count / float(data['updated_drafts_count']),'.0%')
+    except ZeroDivisionError:
+        ffw_update_percent = 0
+
+    data['ffw_update_count'] = ffw_update_count
+    data['ffw_update_percent'] = ffw_update_percent
+
+    rfcs = events.filter(type='published_rfc')
+    data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__intended_std_level')
+
+    data['counts'] = {'std': rfcs.filter(doc__intended_std_level__in=('ps', 'ds', 'std')).count(),
+                      'bcp': rfcs.filter(doc__intended_std_level='bcp').count(),
+                      'exp': rfcs.filter(doc__intended_std_level='exp').count(),
+                      'inf': rfcs.filter(doc__intended_std_level='inf').count()}
+
+    data['new_groups'] = Group.objects.filter(
+        type='wg',
+        groupevent__changestategroupevent__state='active',
+        groupevent__time__gte=sdatetime,
+        groupevent__time__lt=edatetime)
+
+    data['concluded_groups'] = Group.objects.filter(
+        type='wg',
+        groupevent__changestategroupevent__state='conclude',
+        groupevent__time__gte=sdatetime,
+        groupevent__time__lt=edatetime)
+
+    return data
+
+def is_powerpoint(doc):
+    '''
+    Returns true if document is a Powerpoint presentation
+    '''
+    return doc.file_extension() in ('ppt', 'pptx')
+
+def post_process(doc):
+    '''
+    Does post processing on uploaded file.
+    - Convert PPT to PDF
+    '''
+    if is_powerpoint(doc) and hasattr(settings, 'SECR_PPT2PDF_COMMAND'):
+        try:
+            cmd = list(settings.SECR_PPT2PDF_COMMAND)   # Don't operate on the list actually in settings
+            cmd.append(doc.get_file_path())                                 # outdir
+            cmd.append(os.path.join(doc.get_file_path(), doc.uploaded_filename))  # filename
+            subprocess.check_call(cmd)
+        except (subprocess.CalledProcessError, OSError) as error:
+            log("Error converting PPT: %s" % (error))
+            return
+        # change extension
+        base, ext = os.path.splitext(doc.uploaded_filename)
+        doc.uploaded_filename = base + '.pdf'
+
+        e = DocEvent.objects.create(
+            type='changed_document',
+            by=Person.objects.get(name="(System)"),
+            doc=doc,
+            rev=doc.rev,
+            desc='Converted document to PDF',
+        )
+        doc.save_with_history([e])
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
@@ -82,10 +82,9 @@
 from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_create_timeslots
 from ietf.meeting.utils import preprocess_meeting_important_dates
 from ietf.meeting.utils import new_doc_for_session, write_doc_for_session
+from ietf.meeting.utils import get_activity_stats, post_process, create_recording
 from ietf.message.utils import infer_message
 from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
-from ietf.secr.proceedings.proc_utils import (get_activity_stats, post_process, import_audio_files,
-    create_recording)
 from ietf.utils import markdown
 from ietf.utils.decorators import require_api_key
 from ietf.utils.hedgedoc import Note, NoteError
@@ -3798,16 +3797,6 @@ class OldUploadRedirect(RedirectView):
     def get_redirect_url(self, **kwargs):
         return reverse_lazy('ietf.meeting.views.session_details',kwargs=self.kwargs)
 
-@csrf_exempt
-def api_import_recordings(request, number):
-    '''REST API to check for recording files and import'''
-    if request.method == 'POST':
-        meeting = get_meeting(number)
-        import_audio_files(meeting)
-        return HttpResponse(status=201)
-    else:
-        return HttpResponse(status=405)
-
 @require_api_key
 @role_required('Recording Manager')
 @csrf_exempt

diff --git a/ietf/secr/proceedings/__init__.py b/ietf/secr/proceedings/__init__.py
diff --git a/ietf/secr/proceedings/forms.py b/ietf/secr/proceedings/forms.py
diff --git a/ietf/secr/proceedings/migrations/0001_initial.py b/ietf/secr/proceedings/migrations/0001_initial.py
diff --git a/ietf/secr/proceedings/migrations/__init__.py b/ietf/secr/proceedings/migrations/__init__.py