diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 124f3edd55..e11e7eae61 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -8,7 +8,6 @@ import sys from importlib import import_module -from mock import patch from pathlib import Path from django.apps import apps @@ -27,7 +26,6 @@ from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory from ietf.group.factories import RoleFactory from ietf.meeting.factories import MeetingFactory, SessionFactory -from ietf.meeting.test_data import make_meeting_test_data from ietf.meeting.models import Session from ietf.person.factories import PersonFactory, random_faker from ietf.person.models import User @@ -46,20 +44,6 @@ class CustomApiTests(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH'] - # Using mock to patch the import functions in ietf.meeting.views, where - # api_import_recordings() are using them: - @patch('ietf.meeting.views.import_audio_files') - def test_notify_meeting_import_audio_files(self, mock_import_audio): - meeting = make_meeting_test_data() - client = Client(Accept='application/json') - # try invalid method GET - url = urlreverse('ietf.meeting.views.api_import_recordings', kwargs={'number':meeting.number}) - r = client.get(url) - self.assertEqual(r.status_code, 405) - # try valid method POST - r = client.post(url) - self.assertEqual(r.status_code, 201) - def test_api_help_page(self): url = urlreverse('ietf.api.views.api_help') r = self.client.get(url) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index aff73d607d..53b69c05e7 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -32,8 +32,6 @@ url(r'^meeting/(?P[A-Za-z0-9._+-]+)/agenda-data$', meeting_views.api_get_agenda_data), # Meeting session materials url(r'^meeting/session/(?P[A-Za-z0-9._+-]+)/materials$', meeting_views.api_get_session_materials), - # Let Meetecho trigger recording imports - url(r'^notify/meeting/import_recordings/(?P[a-z0-9-]+)/?$', meeting_views.api_import_recordings), # Let MeetEcho upload bluesheets url(r'^notify/meeting/bluesheet/?$', meeting_views.api_upload_bluesheet), # Let MeetEcho tell us about session attendees diff --git a/ietf/doc/management/commands/fix_105_slides.py b/ietf/doc/management/commands/fix_105_slides.py index b8689482e8..377298ea3c 100644 --- a/ietf/doc/management/commands/fix_105_slides.py +++ b/ietf/doc/management/commands/fix_105_slides.py @@ -12,7 +12,7 @@ from ietf.meeting.models import Meeting, SessionPresentation from ietf.person.models import Person -from ietf.secr.proceedings.proc_utils import is_powerpoint, post_process +from ietf.meeting.utils import is_powerpoint, post_process class Command(BaseCommand): help = ('Fix uploaded_filename and generate pdf from pptx') diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index 15675a8327..827f1db840 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -63,7 +63,7 @@ from ietf.iesg.utils import telechat_page_count from ietf.ietfauth.utils import has_role, role_required, user_is_person from ietf.person.models import Person -from ietf.secr.proceedings.proc_utils import get_activity_stats +from ietf.meeting.utils import get_activity_stats from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date from ietf.utils.timezone import date_today, datetime_from_date diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 6109423276..78b7e8346b 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -47,6 +47,7 @@ from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data from ietf.meeting.utils import finalize, condition_slide_order from ietf.meeting.utils import add_event_info_to_session_qs +from ietf.meeting.utils import create_recording, get_next_sequence from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName from ietf.utils.decorators import skip_coverage @@ -8095,3 +8096,20 @@ def test_rename_proceedings_material(self): pm = meeting.proceedings_materials.get(pk=pm.pk) self.assertEqual(str(pm), 'This Is Not the Default Name') self.assertEqual(pm.document.rev, orig_rev, 'Renaming should not change document revision') + + def test_create_recording(self): + session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars') + filename = 'ietf42-testroomt-20000101-0800.mp3' + url = settings.IETF_AUDIO_URL + 'ietf{}/{}'.format(session.meeting.number, filename) + doc = create_recording(session, url) + self.assertEqual(doc.name,'recording-72-mars-1') + self.assertEqual(doc.group,session.group) + self.assertEqual(doc.external_url,url) + self.assertTrue(doc in session.materials.all()) + + def test_get_next_sequence(self): + session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars') + meeting = session.meeting + group = session.group + sequence = get_next_sequence(group,meeting,'recording') + self.assertEqual(sequence,1) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index e8efb92ad0..1f4896c884 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import datetime import itertools +import os import pytz import requests import subprocess @@ -19,13 +20,14 @@ import debug # pyflakes:ignore from ietf.dbtemplate.models import DBTemplate -from ietf.meeting.models import Session, SchedulingEvent, TimeSlot, Constraint, SchedTimeSessAssignment +from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, + Constraint, SchedTimeSessAssignment, SessionPresentation) from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent +from ietf.doc.models import DocEvent from ietf.group.models import Group from ietf.group.utils import can_manage_materials from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName from ietf.person.models import Person -from ietf.secr.proceedings.proc_utils import import_audio_files from ietf.utils.html import sanitize_document from ietf.utils.log import log from ietf.utils.timezone import date_today @@ -180,7 +182,6 @@ def finalize(meeting): sp.rev = '00' sp.save() - import_audio_files(meeting) create_proceedings_templates(meeting) meeting.proceedings_final = True meeting.save() @@ -756,3 +757,156 @@ def write_doc_for_session(session, type_id, filename, contents): with open(path / filename, "wb") as file: file.write(contents.encode('utf-8')) return + +def create_recording(session, url, title=None, user=None): + ''' + Creates the Document type=recording, setting external_url and creating + NewRevisionDocEvent + ''' + sequence = get_next_sequence(session.group,session.meeting,'recording') + name = 'recording-{}-{}-{}'.format(session.meeting.number,session.group.acronym,sequence) + time = session.official_timeslotassignment().timeslot.time.strftime('%Y-%m-%d %H:%M') + if not title: + if url.endswith('mp3'): + title = 'Audio recording for {}'.format(time) + else: + title = 'Video recording for {}'.format(time) + + doc = Document.objects.create(name=name, + title=title, + external_url=url, + group=session.group, + rev='00', + type_id='recording') + doc.set_state(State.objects.get(type='recording', slug='active')) + + DocAlias.objects.create(name=doc.name).docs.add(doc) + + # create DocEvent + NewRevisionDocEvent.objects.create(type='new_revision', + by=user or Person.objects.get(name='(System)'), + doc=doc, + rev=doc.rev, + desc='New revision available', + time=doc.time) + pres = SessionPresentation.objects.create(session=session,document=doc,rev=doc.rev) + session.sessionpresentation_set.add(pres) + + return doc + +def get_next_sequence(group, meeting, type): + ''' + Returns the next sequence number to use for a document of type = type. + Takes a group=Group object, meeting=Meeting object, type = string + ''' + aliases = DocAlias.objects.filter(name__startswith='{}-{}-{}-'.format(type, meeting.number, group.acronym)) + if not aliases: + return 1 + aliases = aliases.order_by('name') + sequence = int(aliases.last().name.split('-')[-1]) + 1 + return sequence + +def get_activity_stats(sdate, edate): + ''' + This function takes a date range and produces a dictionary of statistics / objects for + use in an activity report. Generally the end date will be the date of the last meeting + and the start date will be the date of the meeting before that. + + Data between midnight UTC on the specified dates are included in the stats. + ''' + sdatetime = pytz.utc.localize(datetime.datetime.combine(sdate, datetime.time())) + edatetime = pytz.utc.localize(datetime.datetime.combine(edate, datetime.time())) + + data = {} + data['sdate'] = sdate + data['edate'] = edate + + events = DocEvent.objects.filter(doc__type='draft', time__gte=sdatetime, time__lt=edatetime) + + data['actions_count'] = events.filter(type='iesg_approved').count() + data['last_calls_count'] = events.filter(type='sent_last_call').count() + new_draft_events = events.filter(newrevisiondocevent__rev='00') + new_drafts = list(set([e.doc_id for e in new_draft_events])) + data['new_docs'] = list(set([e.doc for e in new_draft_events])) + data['new_drafts_count'] = len(new_drafts) + data['new_drafts_updated_count'] = events.filter(doc__id__in=new_drafts,newrevisiondocevent__rev='01').count() + data['new_drafts_updated_more_count'] = events.filter(doc__id__in=new_drafts,newrevisiondocevent__rev='02').count() + + update_events = events.filter(type='new_revision').exclude(doc__id__in=new_drafts) + data['updated_drafts_count'] = len(set([e.doc_id for e in update_events])) + + # Calculate Final Four Weeks stats (ffw) + ffwdate = edatetime - datetime.timedelta(days=28) + ffw_new_count = events.filter(time__gte=ffwdate, newrevisiondocevent__rev='00').count() + try: + ffw_new_percent = format(ffw_new_count / float(data['new_drafts_count']), '.0%') + except ZeroDivisionError: + ffw_new_percent = 0 + + data['ffw_new_count'] = ffw_new_count + data['ffw_new_percent'] = ffw_new_percent + + ffw_update_events = events.filter(time__gte=ffwdate, type='new_revision').exclude(doc__id__in=new_drafts) + ffw_update_count = len(set([e.doc_id for e in ffw_update_events])) + try: + ffw_update_percent = format(ffw_update_count / float(data['updated_drafts_count']),'.0%') + except ZeroDivisionError: + ffw_update_percent = 0 + + data['ffw_update_count'] = ffw_update_count + data['ffw_update_percent'] = ffw_update_percent + + rfcs = events.filter(type='published_rfc') + data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__intended_std_level') + + data['counts'] = {'std': rfcs.filter(doc__intended_std_level__in=('ps', 'ds', 'std')).count(), + 'bcp': rfcs.filter(doc__intended_std_level='bcp').count(), + 'exp': rfcs.filter(doc__intended_std_level='exp').count(), + 'inf': rfcs.filter(doc__intended_std_level='inf').count()} + + data['new_groups'] = Group.objects.filter( + type='wg', + groupevent__changestategroupevent__state='active', + groupevent__time__gte=sdatetime, + groupevent__time__lt=edatetime) + + data['concluded_groups'] = Group.objects.filter( + type='wg', + groupevent__changestategroupevent__state='conclude', + groupevent__time__gte=sdatetime, + groupevent__time__lt=edatetime) + + return data + +def is_powerpoint(doc): + ''' + Returns true if document is a Powerpoint presentation + ''' + return doc.file_extension() in ('ppt', 'pptx') + +def post_process(doc): + ''' + Does post processing on uploaded file. + - Convert PPT to PDF + ''' + if is_powerpoint(doc) and hasattr(settings, 'SECR_PPT2PDF_COMMAND'): + try: + cmd = list(settings.SECR_PPT2PDF_COMMAND) # Don't operate on the list actually in settings + cmd.append(doc.get_file_path()) # outdir + cmd.append(os.path.join(doc.get_file_path(), doc.uploaded_filename)) # filename + subprocess.check_call(cmd) + except (subprocess.CalledProcessError, OSError) as error: + log("Error converting PPT: %s" % (error)) + return + # change extension + base, ext = os.path.splitext(doc.uploaded_filename) + doc.uploaded_filename = base + '.pdf' + + e = DocEvent.objects.create( + type='changed_document', + by=Person.objects.get(name="(System)"), + doc=doc, + rev=doc.rev, + desc='Converted document to PDF', + ) + doc.save_with_history([e]) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 68fd84dca9..8dc31a0b39 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -82,10 +82,9 @@ from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_create_timeslots from ietf.meeting.utils import preprocess_meeting_important_dates from ietf.meeting.utils import new_doc_for_session, write_doc_for_session +from ietf.meeting.utils import get_activity_stats, post_process, create_recording from ietf.message.utils import infer_message from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName -from ietf.secr.proceedings.proc_utils import (get_activity_stats, post_process, import_audio_files, - create_recording) from ietf.utils import markdown from ietf.utils.decorators import require_api_key from ietf.utils.hedgedoc import Note, NoteError @@ -3798,16 +3797,6 @@ class OldUploadRedirect(RedirectView): def get_redirect_url(self, **kwargs): return reverse_lazy('ietf.meeting.views.session_details',kwargs=self.kwargs) -@csrf_exempt -def api_import_recordings(request, number): - '''REST API to check for recording files and import''' - if request.method == 'POST': - meeting = get_meeting(number) - import_audio_files(meeting) - return HttpResponse(status=201) - else: - return HttpResponse(status=405) - @require_api_key @role_required('Recording Manager') @csrf_exempt diff --git a/ietf/secr/proceedings/__init__.py b/ietf/secr/proceedings/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ietf/secr/proceedings/forms.py b/ietf/secr/proceedings/forms.py deleted file mode 100644 index 26dd419d66..0000000000 --- a/ietf/secr/proceedings/forms.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved - -from django import forms - -from ietf.doc.models import Document -from ietf.meeting.models import Session -from ietf.meeting.utils import add_event_info_to_session_qs - - -# --------------------------------------------- -# Globals -# --------------------------------------------- - -VALID_SLIDE_EXTENSIONS = ('.doc','.docx','.pdf','.ppt','.pptx','.txt','.zip') -VALID_MINUTES_EXTENSIONS = ('.txt','.html','.htm','.pdf') -VALID_AGENDA_EXTENSIONS = ('.txt','.html','.htm') -VALID_BLUESHEET_EXTENSIONS = ('.pdf','.jpg','.jpeg') - -#---------------------------------------------------------- -# Forms -#---------------------------------------------------------- - -class RecordingForm(forms.Form): - external_url = forms.URLField(label='Url') - session = forms.ModelChoiceField(queryset=Session.objects) - session.widget.attrs['class'] = "select2-field" - session.widget.attrs['data-minimum-input-length'] = 0 - - def __init__(self, *args, **kwargs): - self.meeting = kwargs.pop('meeting') - super(RecordingForm, self).__init__(*args,**kwargs) - self.fields['session'].queryset = add_event_info_to_session_qs( - Session.objects.filter(meeting=self.meeting, type__in=['regular','plenary','other']) - ).filter(current_status='sched').order_by('group__acronym') - -class RecordingEditForm(forms.ModelForm): - class Meta: - model = Document - fields = ['external_url'] - - def __init__(self, *args, **kwargs): - super(RecordingEditForm, self).__init__(*args, **kwargs) - self.fields['external_url'].label='Url' - diff --git a/ietf/secr/proceedings/migrations/0001_initial.py b/ietf/secr/proceedings/migrations/0001_initial.py deleted file mode 100644 index 2aee67d708..0000000000 --- a/ietf/secr/proceedings/migrations/0001_initial.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright The IETF Trust 2018-2020, All Rights Reserved -# -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-20 10:52 - - -from django.db import migrations - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('meeting', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='InterimMeeting', - fields=[ - ], - options={ - 'proxy': True, - 'indexes': [], - }, - bases=('meeting.meeting',), - ), - ] diff --git a/ietf/secr/proceedings/migrations/__init__.py b/ietf/secr/proceedings/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ietf/secr/proceedings/models.py b/ietf/secr/proceedings/models.py deleted file mode 100644 index 75563429e5..0000000000 --- a/ietf/secr/proceedings/models.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import os - -from django.conf import settings -from django.db import models - -from ietf.meeting.models import Meeting - - -class InterimManager(models.Manager): - '''A custom manager to limit objects to type=interim''' - def get_queryset(self): - return super(InterimManager, self).get_queryset().filter(type='interim') - -class InterimMeeting(Meeting): - ''' - This class is a proxy of Meeting. It's purpose is to provide extra methods that are - useful for an interim meeting, to help in templates. Most information is derived from - the session associated with this meeting. We are assuming there is only one. - ''' - class Meta: - proxy = True - - objects = InterimManager() - - def group(self): - return self.session_set.all()[0].group - - def agenda(self): # pylint: disable=method-hidden - session = self.session_set.all()[0] - agendas = session.materials.exclude(states__slug='deleted').filter(type='agenda') - if agendas: - return agendas[0] - else: - return None - - def minutes(self): - session = self.session_set.all()[0] - minutes = session.materials.exclude(states__slug='deleted').filter(type='minutes') - if minutes: - return minutes[0] - else: - return None - - def get_proceedings_path(self, group=None): - return os.path.join(self.get_materials_path(),'proceedings.html') - - def get_proceedings_url(self, group=None): - ''' - If the proceedings file doesn't exist return empty string. For use in templates. - ''' - if os.path.exists(self.get_proceedings_path()): - url = "%sproceedings/%s/proceedings.html" % ( - settings.IETF_HOST_URL, - self.number) - return url - else: - return '' - diff --git a/ietf/secr/proceedings/new_tables.sql b/ietf/secr/proceedings/new_tables.sql deleted file mode 100644 index 43d590f3d8..0000000000 --- a/ietf/secr/proceedings/new_tables.sql +++ /dev/null @@ -1,50 +0,0 @@ -CREATE TABLE `interim_slides` ( - `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, - `meeting_num` integer NOT NULL, - `group_acronym_id` integer, - `slide_num` integer, - `slide_type_id` integer NOT NULL, - `slide_name` varchar(255) NOT NULL, - `irtf` integer NOT NULL, - `interim` bool NOT NULL, - `order_num` integer, - `in_q` integer -) -; -CREATE TABLE `interim_minutes` ( - `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, - `meeting_num` integer NOT NULL, - `group_acronym_id` integer NOT NULL, - `filename` varchar(255) NOT NULL, - `irtf` integer NOT NULL, - `interim` bool NOT NULL -) -; -CREATE TABLE `interim_agenda` ( - `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, - `meeting_num` integer NOT NULL, - `group_acronym_id` integer NOT NULL, - `filename` varchar(255) NOT NULL, - `irtf` integer NOT NULL, - `interim` bool NOT NULL -) -; -CREATE TABLE `interim_meetings` ( - `meeting_num` integer NOT NULL PRIMARY KEY AUTO_INCREMENT, - `start_date` date , - `end_date` date , - `city` varchar(255) , - `state` varchar(255) , - `country` varchar(255) , - `time_zone` integer, - `ack` longtext , - `agenda_html` longtext , - `agenda_text` longtext , - `future_meeting` longtext , - `overview1` longtext , - `overview2` longtext , - `group_acronym_id` integer -) -; -alter table interim_meetings auto_increment=201; - diff --git a/ietf/secr/proceedings/proc_utils.py b/ietf/secr/proceedings/proc_utils.py deleted file mode 100644 index bcba9bfa30..0000000000 --- a/ietf/secr/proceedings/proc_utils.py +++ /dev/null @@ -1,305 +0,0 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -''' -proc_utils.py - -This module contains all the functions for generating static proceedings pages -''' -import datetime -import os -import pytz -import re -import subprocess -from urllib.parse import urlencode - -import debug # pyflakes:ignore - -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist - -from ietf.doc.models import Document, DocAlias, DocEvent, NewRevisionDocEvent, State -from ietf.group.models import Group -from ietf.meeting.models import Meeting, SessionPresentation, TimeSlot, SchedTimeSessAssignment, Session -from ietf.person.models import Person -from ietf.utils.log import log -from ietf.utils.mail import send_mail -from ietf.utils.timezone import make_aware - -AUDIO_FILE_RE = re.compile(r'ietf(?P[\d]+)-(?P.*)-(?P