Skip to content

Commit 8cb7f3d

Browse files
rjsparksrjsparks
authored
feat: Import IESG artifacts into the datatracker (#6908)
* chore: remove unused setting * feat: initial import of iesg minutes * fix: let the meetings view show older iesg meetings * feat: iesg narrative minutes * feat: import bof coordination call minutes * wip: import commands for iesg appeals and statements * feat: import iesg statements. * feat: import iesg artifacts * feat: many fewer n+1 queries for the group meetings view * fix: restore chain of elifs in views_doc * fix: use self.stdout.write vs print in mgmt commands * fix: use replace instead of astimezone when appropriate * chore: refactor new migrations into one * fix: transcode some old files into utf8 * fix: repair overzealous replace * chore: black * fix: address minro review comments * fix: actually capture transcoding work * fix: handle multiple iesg statements on the same day * fix: better titles * feat: pill badge replaced statements * fix: consolodate source repos to one * feat: liberal markdown for secretariat controlled content * fix: handle (and clean) html narrative minutes * feat: scrub harder * fix: simplify and improve a scrubber * chore: reorder migrations
1 parent 5fc0f69 commit 8cb7f3d

31 files changed

+1317
-231
lines changed

‎ietf/api/tests.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ def test_api_upload_polls_and_chatlog(self):
368368
r = self.client.post(url,{'apikey':apikey.hash(),'apidata': f'{{"session_id":{session.pk}, "{type_id}":{content}}}'})
369369
self.assertEqual(r.status_code, 200)
370370

371-
newdoc = session.sessionpresentation_set.get(document__type_id=type_id).document
371+
newdoc = session.presentations.get(document__type_id=type_id).document
372372
newdoccontent = get_unicode_document_content(newdoc.name, Path(session.meeting.get_materials_path()) / type_id / newdoc.uploaded_filename)
373373
self.assertEqual(json.loads(content), json.loads(newdoccontent))
374374

@@ -454,7 +454,7 @@ def test_deprecated_api_upload_bluesheet(self):
454454
'item': '1', 'bluesheet': bluesheet, })
455455
self.assertContains(r, "Done", status_code=200)
456456

457-
bluesheet = session.sessionpresentation_set.filter(document__type__slug='bluesheets').first().document
457+
bluesheet = session.presentations.filter(document__type__slug='bluesheets').first().document
458458
# We've submitted an update; check that the rev is right
459459
self.assertEqual(bluesheet.rev, '01')
460460
# Check the content
@@ -569,7 +569,7 @@ def test_api_upload_bluesheet(self):
569569
self.assertContains(r, "Done", status_code=200)
570570

571571
bluesheet = (
572-
session.sessionpresentation_set.filter(document__type__slug="bluesheets")
572+
session.presentations.filter(document__type__slug="bluesheets")
573573
.first()
574574
.document
575575
)

‎ietf/doc/migrations/0021_narrativeminutes.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright The IETF Trust 2023, All Rights Reserved
2+
3+
from django.db import migrations
4+
5+
6+
def forward(apps, schema_editor):
7+
StateType = apps.get_model("doc", "StateType")
8+
State = apps.get_model("doc", "State")
9+
10+
StateType.objects.create(
11+
slug="narrativeminutes",
12+
label="State",
13+
)
14+
for order, slug in enumerate(["active", "deleted"]):
15+
State.objects.create(
16+
slug=slug,
17+
type_id="narrativeminutes",
18+
name=slug.capitalize(),
19+
order=order,
20+
desc="",
21+
used=True,
22+
)
23+
24+
25+
def reverse(apps, schema_editor):
26+
StateType = apps.get_model("doc", "StateType")
27+
State = apps.get_model("doc", "State")
28+
29+
State.objects.filter(type_id="narrativeminutes").delete()
30+
StateType.objects.filter(slug="narrativeminutes").delete()
31+
32+
33+
class Migration(migrations.Migration):
34+
dependencies = [
35+
("doc", "0020_move_errata_tags"),
36+
("name", "0013_narrativeminutes"),
37+
]
38+
39+
operations = [migrations.RunPython(forward, reverse)]

‎ietf/doc/models.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ def get_file_path(self):
148148
else:
149149
self._cached_file_path = settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR
150150
elif self.meeting_related() and self.type_id in (
151-
"agenda", "minutes", "slides", "bluesheets", "procmaterials", "chatlog", "polls"
151+
"agenda", "minutes", "narrativeminutes", "slides", "bluesheets", "procmaterials", "chatlog", "polls"
152152
):
153153
meeting = self.get_related_meeting()
154154
if meeting is not None:
@@ -438,7 +438,7 @@ def has_rfc_editor_note(self):
438438
return e != None and (e.text != "")
439439

440440
def meeting_related(self):
441-
if self.type_id in ("agenda","minutes","bluesheets","slides","recording","procmaterials","chatlog","polls"):
441+
if self.type_id in ("agenda","minutes", "narrativeminutes", "bluesheets","slides","recording","procmaterials","chatlog","polls"):
442442
return self.type_id != "slides" or self.get_state_slug('reuse_policy')=='single'
443443
return False
444444

@@ -1028,7 +1028,7 @@ def related_ipr(self):
10281028
def future_presentations(self):
10291029
""" returns related SessionPresentation objects for meetings that
10301030
have not yet ended. This implementation allows for 2 week meetings """
1031-
candidate_presentations = self.sessionpresentation_set.filter(
1031+
candidate_presentations = self.presentations.filter(
10321032
session__meeting__date__gte=date_today() - datetime.timedelta(days=15)
10331033
)
10341034
return sorted(
@@ -1041,11 +1041,11 @@ def last_presented(self):
10411041
""" returns related SessionPresentation objects for the most recent meeting in the past"""
10421042
# Assumes no two meetings have the same start date - if the assumption is violated, one will be chosen arbitrarily
10431043
today = date_today()
1044-
candidate_presentations = self.sessionpresentation_set.filter(session__meeting__date__lte=today)
1044+
candidate_presentations = self.presentations.filter(session__meeting__date__lte=today)
10451045
candidate_meetings = set([p.session.meeting for p in candidate_presentations if p.session.meeting.end_date()<today])
10461046
if candidate_meetings:
10471047
mtg = sorted(list(candidate_meetings),key=lambda x:x.date,reverse=True)[0]
1048-
return self.sessionpresentation_set.filter(session__meeting=mtg)
1048+
return self.presentations.filter(session__meeting=mtg)
10491049
else:
10501050
return None
10511051

‎ietf/doc/tests.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2529,8 +2529,8 @@ def setUp(self):
25292529

25302530
def test_view_document_meetings(self):
25312531
doc = IndividualDraftFactory.create()
2532-
doc.sessionpresentation_set.create(session=self.inprog,rev=None)
2533-
doc.sessionpresentation_set.create(session=self.interim,rev=None)
2532+
doc.presentations.create(session=self.inprog,rev=None)
2533+
doc.presentations.create(session=self.interim,rev=None)
25342534

25352535
url = urlreverse('ietf.doc.views_doc.all_presentations', kwargs=dict(name=doc.name))
25362536
response = self.client.get(url)
@@ -2541,8 +2541,8 @@ def test_view_document_meetings(self):
25412541
self.assertFalse(q('#addsessionsbutton'))
25422542
self.assertFalse(q("a.btn:contains('Remove document')"))
25432543

2544-
doc.sessionpresentation_set.create(session=self.past_cutoff,rev=None)
2545-
doc.sessionpresentation_set.create(session=self.past,rev=None)
2544+
doc.presentations.create(session=self.past_cutoff,rev=None)
2545+
doc.presentations.create(session=self.past,rev=None)
25462546

25472547
self.client.login(username="secretary", password="secretary+password")
25482548
response = self.client.get(url)
@@ -2577,7 +2577,7 @@ def test_view_document_meetings(self):
25772577

25782578
def test_edit_document_session(self):
25792579
doc = IndividualDraftFactory.create()
2580-
sp = doc.sessionpresentation_set.create(session=self.future,rev=None)
2580+
sp = doc.presentations.create(session=self.future,rev=None)
25812581

25822582
url = urlreverse('ietf.doc.views_doc.edit_sessionpresentation',kwargs=dict(name='no-such-doc',session_id=sp.session_id))
25832583
response = self.client.get(url)
@@ -2604,12 +2604,12 @@ def test_edit_document_session(self):
26042604
self.assertEqual(1,doc.docevent_set.count())
26052605
response = self.client.post(url,{'version':'00','save':''})
26062606
self.assertEqual(response.status_code, 302)
2607-
self.assertEqual(doc.sessionpresentation_set.get(pk=sp.pk).rev,'00')
2607+
self.assertEqual(doc.presentations.get(pk=sp.pk).rev,'00')
26082608
self.assertEqual(2,doc.docevent_set.count())
26092609

26102610
def test_edit_document_session_after_proceedings_closed(self):
26112611
doc = IndividualDraftFactory.create()
2612-
sp = doc.sessionpresentation_set.create(session=self.past_cutoff,rev=None)
2612+
sp = doc.presentations.create(session=self.past_cutoff,rev=None)
26132613

26142614
url = urlreverse('ietf.doc.views_doc.edit_sessionpresentation',kwargs=dict(name=doc.name,session_id=sp.session_id))
26152615
self.client.login(username=self.group_chair.user.username,password='%s+password'%self.group_chair.user.username)
@@ -2624,7 +2624,7 @@ def test_edit_document_session_after_proceedings_closed(self):
26242624

26252625
def test_remove_document_session(self):
26262626
doc = IndividualDraftFactory.create()
2627-
sp = doc.sessionpresentation_set.create(session=self.future,rev=None)
2627+
sp = doc.presentations.create(session=self.future,rev=None)
26282628

26292629
url = urlreverse('ietf.doc.views_doc.remove_sessionpresentation',kwargs=dict(name='no-such-doc',session_id=sp.session_id))
26302630
response = self.client.get(url)
@@ -2649,12 +2649,12 @@ def test_remove_document_session(self):
26492649
self.assertEqual(1,doc.docevent_set.count())
26502650
response = self.client.post(url,{'remove_session':''})
26512651
self.assertEqual(response.status_code, 302)
2652-
self.assertFalse(doc.sessionpresentation_set.filter(pk=sp.pk).exists())
2652+
self.assertFalse(doc.presentations.filter(pk=sp.pk).exists())
26532653
self.assertEqual(2,doc.docevent_set.count())
26542654

26552655
def test_remove_document_session_after_proceedings_closed(self):
26562656
doc = IndividualDraftFactory.create()
2657-
sp = doc.sessionpresentation_set.create(session=self.past_cutoff,rev=None)
2657+
sp = doc.presentations.create(session=self.past_cutoff,rev=None)
26582658

26592659
url = urlreverse('ietf.doc.views_doc.remove_sessionpresentation',kwargs=dict(name=doc.name,session_id=sp.session_id))
26602660
self.client.login(username=self.group_chair.user.username,password='%s+password'%self.group_chair.user.username)

‎ietf/doc/views_doc.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -832,7 +832,7 @@ def document_main(request, name, rev=None, document_html=False):
832832
sorted_relations=sorted_relations,
833833
))
834834

835-
elif doc.type_id in ("slides", "agenda", "minutes", "bluesheets", "procmaterials",):
835+
elif doc.type_id in ("slides", "agenda", "minutes", "narrativeminutes", "bluesheets", "procmaterials",):
836836
can_manage_material = can_manage_materials(request.user, doc.group)
837837
presentations = doc.future_presentations()
838838
if doc.uploaded_filename:
@@ -916,9 +916,9 @@ def document_main(request, name, rev=None, document_html=False):
916916

917917
elif doc.type_id in ("chatlog", "polls"):
918918
if isinstance(doc,DocHistory):
919-
session = doc.doc.sessionpresentation_set.last().session
919+
session = doc.doc.presentations.last().session
920920
else:
921-
session = doc.sessionpresentation_set.last().session
921+
session = doc.presentations.last().session
922922
pathname = Path(session.meeting.get_materials_path()) / doc.type_id / doc.uploaded_filename
923923
content = get_unicode_document_content(doc.name, str(pathname))
924924
return render(
@@ -943,7 +943,7 @@ def document_main(request, name, rev=None, document_html=False):
943943
variants = set([match.name.split(".")[1] for match in Path(doc.get_file_path()).glob(f"{basename}.*")])
944944
inlineable = any([ext in variants for ext in ["md", "txt"]])
945945
if inlineable:
946-
content = markdown.markdown(doc.text_or_error())
946+
content = markdown.liberal_markdown(doc.text_or_error())
947947
else:
948948
content = "No format available to display inline"
949949
if "pdf" in variants:
@@ -2057,7 +2057,7 @@ def __init__(self, *args, **kwargs):
20572057

20582058
def edit_sessionpresentation(request,name,session_id):
20592059
doc = get_object_or_404(Document, name=name)
2060-
sp = get_object_or_404(doc.sessionpresentation_set, session_id=session_id)
2060+
sp = get_object_or_404(doc.presentations, session_id=session_id)
20612061

20622062
if not sp.session.can_manage_materials(request.user):
20632063
raise Http404
@@ -2074,7 +2074,7 @@ def edit_sessionpresentation(request,name,session_id):
20742074
if form.is_valid():
20752075
new_selection = form.cleaned_data['version']
20762076
if initial['version'] != new_selection:
2077-
doc.sessionpresentation_set.filter(pk=sp.pk).update(rev=None if new_selection=='current' else new_selection)
2077+
doc.presentations.filter(pk=sp.pk).update(rev=None if new_selection=='current' else new_selection)
20782078
c = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=request.user.person)
20792079
c.desc = "Revision for session %s changed to %s" % (sp.session,new_selection)
20802080
c.save()
@@ -2086,7 +2086,7 @@ def edit_sessionpresentation(request,name,session_id):
20862086

20872087
def remove_sessionpresentation(request,name,session_id):
20882088
doc = get_object_or_404(Document, name=name)
2089-
sp = get_object_or_404(doc.sessionpresentation_set, session_id=session_id)
2089+
sp = get_object_or_404(doc.presentations, session_id=session_id)
20902090

20912091
if not sp.session.can_manage_materials(request.user):
20922092
raise Http404
@@ -2095,7 +2095,7 @@ def remove_sessionpresentation(request,name,session_id):
20952095
raise Http404
20962096

20972097
if request.method == 'POST':
2098-
doc.sessionpresentation_set.filter(pk=sp.pk).delete()
2098+
doc.presentations.filter(pk=sp.pk).delete()
20992099
c = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=request.user.person)
21002100
c.desc = "Removed from session: %s" % (sp.session)
21012101
c.save()
@@ -2119,7 +2119,7 @@ def add_sessionpresentation(request,name):
21192119
version_choices.insert(0,('current','Current at the time of the session'))
21202120

21212121
sessions = get_upcoming_manageable_sessions(request.user)
2122-
sessions = sort_sessions([s for s in sessions if not s.sessionpresentation_set.filter(document=doc).exists()])
2122+
sessions = sort_sessions([s for s in sessions if not s.presentations.filter(document=doc).exists()])
21232123
if doc.group:
21242124
sessions = sorted(sessions,key=lambda x:0 if x.group==doc.group else 1)
21252125

@@ -2132,7 +2132,7 @@ def add_sessionpresentation(request,name):
21322132
session_id = session_form.cleaned_data['session']
21332133
version = version_form.cleaned_data['version']
21342134
rev = None if version=='current' else version
2135-
doc.sessionpresentation_set.create(session_id=session_id,rev=rev)
2135+
doc.presentations.create(session_id=session_id,rev=rev)
21362136
c = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=request.user.person)
21372137
c.desc = "%s to session: %s" % ('Added -%s'%rev if rev else 'Added', Session.objects.get(pk=session_id))
21382138
c.save()

‎ietf/doc/views_material.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
113113
valid_doctypes = ['procmaterials']
114114
if group is not None:
115115
valid_doctypes.extend(['minutes','agenda','bluesheets'])
116+
if group.acronym=="iesg":
117+
valid_doctypes.append("narrativeminutes")
116118
valid_doctypes.extend(group.features.material_types)
117119

118120
if document_type.slug not in valid_doctypes:

‎ietf/doc/views_statement.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def require_field(f):
9494
)
9595
if markdown_content != "":
9696
try:
97-
_ = markdown.markdown(markdown_content)
97+
_ = markdown.liberal_markdown(markdown_content)
9898
except Exception as e:
9999
raise forms.ValidationError(f"Markdown processing failed: {e}")
100100

0 commit comments

Comments
 (0)