From 2b069edc6d9550e4732ea3f35d7730d62bb96e10 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Wed, 21 Sep 2016 01:01:45 +0200 Subject: [PATCH 01/13] Save VTIMEZONE when importing. --- khal/controllers.py | 35 ++++---- khal/utils.py | 64 +++++++++++--- tests/ics/cal_lots_of_timezones.ics | 128 ++++++++++++++++++++++++++++ tests/ics/part0.ics | 94 ++++++++++++++++++++ tests/ics/part1.ics | 28 ++++++ tests/utils_test.py | 35 ++++---- 6 files changed, 338 insertions(+), 46 deletions(-) create mode 100644 tests/ics/cal_lots_of_timezones.ics create mode 100644 tests/ics/part0.ics create mode 100644 tests/ics/part1.ics diff --git a/khal/controllers.py b/khal/controllers.py index f622e7b7e..054769565 100644 --- a/khal/controllers.py +++ b/khal/controllers.py @@ -528,27 +528,28 @@ def import_ics(collection, conf, ics, batch=False, random_uid=False, format=None even when an event with the same uid already exists :type batch: bool """ - cal = icalendar.Calendar.from_ical(ics) - events = [item for item in cal.walk() if item.name == 'VEVENT'] - events_grouped = defaultdict(list) - for event in events: - events_grouped[event['UID']].append(event) - if format is None: format = conf['view']['event_format'] - vevents = list() - for uid in events_grouped: - vevents.append(sorted(events_grouped[uid], key=sort_key)) + vevents = utils.split_ics(ics) + for vevent in vevents: - import_event(vevent, collection, conf['locale'], batch, random_uid, format, env) + import_event( + vevent, collection, conf['locale'], batch, random_uid, format, env, + ) def import_event(vevent, collection, locale, batch, random_uid, format=None, env=None): - """import one event into collection, let user choose the collection""" + """import one event into collection, let user choose the collection + :type vevent: list of vevents, which can be more than one VEVENT, i.e., the + same UID, i.e., one "master" event and (optionally) 1+ RECURRENCE-ID events + :type vevent: list(str) + """ + # TODO reenbale random_uid # print all sub-events - for sub_event in vevent: + cal = icalendar.Calendar.from_ical(vevent) + for sub_event in [item for item in cal.walk() if item.name == 'VEVENT']: if not batch: event = Event.fromVEvents( [sub_event], calendar=collection.default_calendar_name, locale=locale) @@ -577,13 +578,11 @@ def import_event(vevent, collection, locale, batch, random_uid, format=None, env echo('invalid choice') if batch or confirm("Do you want to import this event into `{}`?".format(calendar_name)): - ics = utils.ics_from_list(vevent, random_uid) try: - collection.new(Item(ics.to_ical().decode('utf-8')), collection=calendar_name) + collection.new(Item(vevent), collection=calendar_name) except DuplicateUid: - if batch or confirm(u"An event with the same UID already exists. " - u"Do you want to update it?"): - collection.force_update( - Item(ics.to_ical().decode('utf-8')), collection=calendar_name) + if batch or confirm( + "An event with the same UID already exists. Do you want to update it?"): + collection.force_update(Item(vevent), collection=calendar_name) else: logger.warn(u"Not importing event with UID `{}`".format(event.uid)) diff --git a/khal/utils.py b/khal/utils.py index 5164421ea..0711b5f12 100644 --- a/khal/utils.py +++ b/khal/utils.py @@ -23,6 +23,7 @@ strings to date(time) or event objects""" from calendar import isleap +from collections import defaultdict from datetime import date, datetime, timedelta, time import random import string @@ -590,19 +591,62 @@ def new_event(locale, dtstart=None, dtend=None, summary=None, timezone=None, return event -def ics_from_list(vevent, random_uid=False): - """convert an iterable of icalendar.Event to an icalendar.Calendar +def split_ics(ics): + """split an ics string into several according to VEVENT's UIDs - :param random_uid: asign the same random UID to all events - :type random_uid: bool + and sort the right VTIMEZONEs accordingly + ignores all other ics components + :type ics: str + :rtype list: + """ + cal = icalendar.Calendar.from_ical(ics) + vevents = [item for item in cal.walk() if item.name == 'VEVENT'] + tzs = {item['TZID']: item for item in cal.walk() if item.name == 'VTIMEZONE'} + + events_grouped = defaultdict(list) + for event in vevents: + events_grouped[event['UID']].append(event) + return [ics_from_list(events, tzs) for uid, events in sorted(events_grouped.items())] + + +def ics_from_list(events, tzs): + """convert an iterable of icalendar.Events to an icalendar.Calendar + + :params events: list of events all with the same uid + :type events: list(icalendar.cal.Event) + :param tzs: collection of timezones + :type tzs: dict(icalendar.cal.Vtimzone """ calendar = icalendar.Calendar() calendar.add('version', '2.0') calendar.add('prodid', '-//CALENDARSERVER.ORG//NONSGML Version 1//EN') - if random_uid: - new_uid = icalendar.vText(generate_random_uid()) - for sub_event in vevent: - if random_uid: - sub_event['uid'] = new_uid + + needed_tz = set() + for sub_event in events: + # RRULE-UNTIL XXX + # icalendar roundrip converts `TZID=a b` to `TZID="a b"` investigate, file bug XXX + for prop in ['DTSTART', 'DTEND', 'DUE', 'EXDATE', 'RDATE', 'RECURRENCE-ID', 'DUE']: + if isinstance(sub_event.get(prop), list): + items = sub_event.get(prop) + else: + items = [sub_event.get(prop)] + for item in items: + try: + # if prop is a list, they all have the same parameters + if hasattr(item, 'dts'): + datetime_ = item.dts[0].dt + else: + datetime_ = item.dt + needed_tz.add(datetime_.tzinfo) + except AttributeError: + continue + + for tzid in needed_tz: + if str(tzid) in tzs: + calendar.add_component(tzs[str(tzid)]) + else: + logger.warn( + 'Cannot find timezone `{}` in .ics file'.format(tzid)) # XXX + for sub_event in events: calendar.add_component(sub_event) - return calendar + return calendar.to_ical().decode('utf-8') diff --git a/tests/ics/cal_lots_of_timezones.ics b/tests/ics/cal_lots_of_timezones.ics new file mode 100644 index 000000000..e175f042a --- /dev/null +++ b/tests/ics/cal_lots_of_timezones.ics @@ -0,0 +1,128 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN +BEGIN:VTIMEZONE +TZID:IndianReunion +BEGIN:STANDARD +TZOFFSETFROM:+034152 +TZOFFSETTO:+0400 +TZNAME:RET +DTSTART:19110601T000000 +RDATE:19110601T000000 +END:STANDARD +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Will_not_appear +BEGIN:STANDARD +TZOFFSETFROM:+034152 +TZOFFSETTO:+0400 +TZNAME:RET +DTSTART:19110601T000000 +RDATE:19110601T000000 +END:STANDARD +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Europe_Amsterdam +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19810329T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19961027T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Europe_Berlin +BEGIN:STANDARD +DTSTART;VALUE=DATE-TIME:20141026T020000 +TZNAME:CET +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +RDATE:20151025T020000 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART;VALUE=DATE-TIME:20140330T030000 +RDATE:20150329T030000,20160327T030000 +TZNAME:CEST +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:America_New_York +BEGIN:STANDARD +DTSTART;VALUE=DATE-TIME:20141102T010000 +RDATE:20151101T010000 +TZNAME:EST +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART;VALUE=DATE-TIME:20140309T030000 +RDATE:20150308T030000,20160313T030000 +TZNAME:EDT +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:America_Bogota +BEGIN:STANDARD +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +TZNAME:COT +DTSTART:19930404T000000 +RDATE:19930404T000000 +END:STANDARD +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Europe_London +BEGIN:DAYLIGHT +TZOFFSETFROM:+0000 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19810329T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0100 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:19961027T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +SUMMARY:An Event +DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 +DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 +RDATE;TZID=America_Bogota:20140411T113000,20140413T113000 +RDATE;TZID=America_Bogota:20140415T113000 +RDATE;TZID=IndianReunion:20140418T113000 +RRULE:FREQ=MONTHLY;COUNT=6 +DTSTAMP;VALUE=DATE-TIME:20140401T234817Z +UID:abcde +END:VEVENT +BEGIN:VEVENT +SUMMARY:An Event +DTSTART;TZID=Europe_London;VALUE=DATE-TIME:20140509T193000 +DTEND;TZID=Europe_London;VALUE=DATE-TIME:20140509T203000 +DTSTAMP;VALUE=DATE-TIME:20140401T234817Z +UID:123 +END:VEVENT +BEGIN:VEVENT +SUMMARY:An Updated Event +DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 +DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 +DTSTAMP;VALUE=DATE-TIME:20140401T234817Z +UID:abcde +RECURRENCE-ID;TZID=Europe_Amsterdam:20140707T070000 +END:VEVENT +END:VCALENDAR diff --git a/tests/ics/part0.ics b/tests/ics/part0.ics new file mode 100644 index 000000000..1a449b0ab --- /dev/null +++ b/tests/ics/part0.ics @@ -0,0 +1,94 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN +BEGIN:VTIMEZONE +TZID:IndianReunion +BEGIN:STANDARD +TZOFFSETFROM:+034152 +TZOFFSETTO:+0400 +TZNAME:RET +DTSTART:19110601T000000 +RDATE:19110601T000000 +END:STANDARD +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Europe_Amsterdam +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19810329T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19961027T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Europe_Berlin +BEGIN:STANDARD +DTSTART;VALUE=DATE-TIME:20141026T020000 +TZNAME:CET +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +RDATE:20151025T020000 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART;VALUE=DATE-TIME:20140330T030000 +RDATE:20150329T030000,20160327T030000 +TZNAME:CEST +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:America_New_York +BEGIN:STANDARD +DTSTART;VALUE=DATE-TIME:20141102T010000 +RDATE:20151101T010000 +TZNAME:EST +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART;VALUE=DATE-TIME:20140309T030000 +RDATE:20150308T030000,20160313T030000 +TZNAME:EDT +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:America_Bogota +BEGIN:STANDARD +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +TZNAME:COT +DTSTART:19930404T000000 +RDATE:19930404T000000 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +SUMMARY:An Event +DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 +DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 +RDATE;TZID=IndianReunion:20140418T113000 +RDATE;TZID=America_Bogota:20140411T113000,20140413T113000 +RDATE;TZID=America_Bogota:20140415T113000 +RRULE:FREQ=MONTHLY;COUNT=6 +DTSTAMP;VALUE=DATE-TIME:20140401T234817Z +UID:abcde +END:VEVENT +BEGIN:VEVENT +SUMMARY:An Updated Event +DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 +DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 +DTSTAMP;VALUE=DATE-TIME:20140401T234817Z +UID:abcde +RECURRENCE-ID;TZID=Europe_Amsterdam:20140707T070000 +END:VEVENT +END:VCALENDAR diff --git a/tests/ics/part1.ics b/tests/ics/part1.ics new file mode 100644 index 000000000..b919d4819 --- /dev/null +++ b/tests/ics/part1.ics @@ -0,0 +1,28 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN +BEGIN:VTIMEZONE +TZID:Europe_London +BEGIN:DAYLIGHT +TZOFFSETFROM:+0000 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19810329T010000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0100 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:19961027T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +SUMMARY:An Event +DTSTART;TZID=Europe_London;VALUE=DATE-TIME:20140509T193000 +DTEND;TZID=Europe_London;VALUE=DATE-TIME:20140509T203000 +DTSTAMP;VALUE=DATE-TIME:20140401T234817Z +UID:123 +END:VEVENT +END:VCALENDAR diff --git a/tests/utils_test.py b/tests/utils_test.py index 9a30f13dd..38baf6de7 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -508,21 +508,20 @@ def test_description_and_location_and_categories(): assert _replace_uid(event).to_ical() == vevent -class TestIcsFromList(object): - - def test_ics_from_list(self): - vevents = _get_all_vevents_file('event_rrule_recuid') - cal = utils.ics_from_list(list(vevents)) - assert normalize_component(cal.to_ical()) == \ - normalize_component(_get_text('event_rrule_recuid')) - - def test_ics_from_list_random_uid(self): - vevents = _get_all_vevents_file('event_rrule_recuid') - cal = utils.ics_from_list(list(vevents), random_uid=True) - normalize_component(cal.to_ical()) - vevents = [item for item in cal.walk() if item.name == 'VEVENT'] - uids = set() - for event in vevents: - uids.add(event['UID']) - assert len(uids) == 1 - assert event['UID'] != icalendar.vText('event_rrule_recurrence_id') +def test_split_ics(): + cal = _get_text('cal_lots_of_timezones') + vevents = utils.split_ics(cal) + + vevents0 = vevents[0].split('\r\n') + vevents1 = vevents[1].split('\r\n') + + part0 = _get_text('part1').split('\n') + part1 = _get_text('part0').split('\n') + + assert sorted([line for line in vevents1 if line.startswith('TZID')]) == \ + sorted([line for line in part1 if line.startswith('TZID')]) + assert sorted([line for line in vevents0 if line.startswith('TZID')]) == \ + sorted([line for line in part0 if line.startswith('TZID')]) + + assert sorted(vevents0) == sorted(part0) + assert sorted(vevents1) == sorted(part1) From 07a2b5b828dec64f07874d74e54556c42d07d82a Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Fri, 30 Sep 2016 17:19:57 +0200 Subject: [PATCH 02/13] typos & doc strings --- khal/controllers.py | 3 ++- khal/khalendar/utils.py | 13 +++++++------ khal/utils.py | 8 +++++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/khal/controllers.py b/khal/controllers.py index 054769565..294fcc948 100644 --- a/khal/controllers.py +++ b/khal/controllers.py @@ -564,7 +564,8 @@ def import_event(vevent, collection, locale, batch, random_uid, format=None, env ['{}({})'.format(name, num) for num, name in enumerate(calendar_names)]) while True: value = prompt( - 'Which calendar do you want to import to? \n{}'.format(choices), + 'Which calendar do you want to import to? (unique prefixes are fine)\n' + '{}'.format(choices), default=collection.default_calendar_name, ) try: diff --git a/khal/khalendar/utils.py b/khal/khalendar/utils.py index 8ac94317b..66f3f4e1f 100644 --- a/khal/khalendar/utils.py +++ b/khal/khalendar/utils.py @@ -121,8 +121,8 @@ def sanitize(vevent, default_timezone, href='', calendar=''): clean up vevents we do not understand :param vevent: the vevent that needs to be cleaned - :type vevent: icalendar.cal.event - :param default_timezone: timezone to apply to stard and/or end dates which + :type vevent: icalendar.cal.Event + :param default_timezone: timezone to apply to start and/or end dates which were supposed to be localized but which timezone was not understood by icalendar :type timezone: pytz.timezone @@ -133,7 +133,7 @@ def sanitize(vevent, default_timezone, href='', calendar=''): problematic :type calendar: str :returns: clean vevent - :rtype: icalendar.cal.event + :rtype: icalendar.cal.Event """ # convert localized datetimes with timezone information we don't # understand to the default timezone @@ -143,8 +143,9 @@ def sanitize(vevent, default_timezone, href='', calendar=''): if prop in vevent and invalid_timezone(vevent[prop]): value = default_timezone.localize(vevent.pop(prop).dt) vevent.add(prop, value) - logger.warn('{} has invalid or incomprehensible timezone ' - 'information in {} in {}'.format(prop, href, calendar)) + logger.warn( + '{} has invalid or incomprehensible timezone ' + 'information in {} in {}'.format(prop, href, calendar)) vdtstart = vevent.pop('DTSTART', None) vdtend = vevent.pop('DTEND', None) @@ -236,7 +237,7 @@ def to_naive_utc(dtime): def invalid_timezone(prop): - """check if a icalendar property has a timezone attached we don't understand""" + """check if an icalendar property has a timezone attached we don't understand""" if hasattr(prop.dt, 'tzinfo') and prop.dt.tzinfo is None and 'TZID' in prop.params: return True else: diff --git a/khal/utils.py b/khal/utils.py index 0711b5f12..25bb90901 100644 --- a/khal/utils.py +++ b/khal/utils.py @@ -38,12 +38,14 @@ def timefstr(dtime_list, timeformat): - """converts a time (as a string) to a datetimeobject + """converts the first item of a list (a time as a string) to a datetimeobject - the date is today + where the date is today and the time is given by the a string removes "used" elements of list - :returns: datetimeobject + :type dtime_list: list(str) + :type timeformat: str + :rtype: datetime.datetime """ if len(dtime_list) == 0: raise ValueError() From fb39dd9275845a116dedd41b434a0656146a71c2 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Fri, 30 Sep 2016 20:05:40 +0200 Subject: [PATCH 03/13] typos --- khal/controllers.py | 8 ++++---- khal/utils.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/khal/controllers.py b/khal/controllers.py index 294fcc948..6f8771125 100644 --- a/khal/controllers.py +++ b/khal/controllers.py @@ -546,7 +546,7 @@ def import_event(vevent, collection, locale, batch, random_uid, format=None, env same UID, i.e., one "master" event and (optionally) 1+ RECURRENCE-ID events :type vevent: list(str) """ - # TODO reenbale random_uid + # TODO re-enable random_uid # print all sub-events cal = icalendar.Calendar.from_ical(vevent) for sub_event in [item for item in cal.walk() if item.name == 'VEVENT']: @@ -564,8 +564,8 @@ def import_event(vevent, collection, locale, batch, random_uid, format=None, env ['{}({})'.format(name, num) for num, name in enumerate(calendar_names)]) while True: value = prompt( - 'Which calendar do you want to import to? (unique prefixes are fine)\n' - '{}'.format(choices), + "Which calendar do you want to import to? (unique prefixes are fine)\n" + "{}".format(choices), default=collection.default_calendar_name, ) try: @@ -586,4 +586,4 @@ def import_event(vevent, collection, locale, batch, random_uid, format=None, env "An event with the same UID already exists. Do you want to update it?"): collection.force_update(Item(vevent), collection=calendar_name) else: - logger.warn(u"Not importing event with UID `{}`".format(event.uid)) + logger.warn("Not importing event with UID `{}`".format(event.uid)) diff --git a/khal/utils.py b/khal/utils.py index 25bb90901..ae144d86d 100644 --- a/khal/utils.py +++ b/khal/utils.py @@ -625,8 +625,8 @@ def ics_from_list(events, tzs): needed_tz = set() for sub_event in events: - # RRULE-UNTIL XXX - # icalendar roundrip converts `TZID=a b` to `TZID="a b"` investigate, file bug XXX + # RRULE-UNTIL XXX can that even be anything but UTC? + # icalendar round-trip converts `TZID=a b` to `TZID="a b"` investigate, file bug XXX for prop in ['DTSTART', 'DTEND', 'DUE', 'EXDATE', 'RDATE', 'RECURRENCE-ID', 'DUE']: if isinstance(sub_event.get(prop), list): items = sub_event.get(prop) @@ -634,7 +634,7 @@ def ics_from_list(events, tzs): items = [sub_event.get(prop)] for item in items: try: - # if prop is a list, they all have the same parameters + # if prop is a list, all items have the same parameters if hasattr(item, 'dts'): datetime_ = item.dts[0].dt else: From e550daec536fbd1d6f6f58567876d896009fe98e Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Fri, 30 Sep 2016 20:12:36 +0200 Subject: [PATCH 04/13] Refactoring importing. --- khal/controllers.py | 12 +++--- khal/utils.py | 42 ++++++++++++++------- tests/ics/part0.ics | 90 ++++++--------------------------------------- tests/ics/part1.ics | 90 +++++++++++++++++++++++++++++++++++++++------ 4 files changed, 124 insertions(+), 110 deletions(-) diff --git a/khal/controllers.py b/khal/controllers.py index 6f8771125..bc1c896d1 100644 --- a/khal/controllers.py +++ b/khal/controllers.py @@ -548,12 +548,12 @@ def import_event(vevent, collection, locale, batch, random_uid, format=None, env """ # TODO re-enable random_uid # print all sub-events - cal = icalendar.Calendar.from_ical(vevent) - for sub_event in [item for item in cal.walk() if item.name == 'VEVENT']: - if not batch: - event = Event.fromVEvents( - [sub_event], calendar=collection.default_calendar_name, locale=locale) - echo(event.format(format, datetime.now(), env=env)) + if not batch: + for item in icalendar.Calendar.from_ical(vevent).walk(): + if item.name == 'VEVENT': + event = Event.fromVEvents( + [item], calendar=collection.default_calendar_name, locale=locale) + echo(event.format(format, datetime.now(), env=env)) # get the calendar to insert into if batch or len(collection.writable_names) == 1: diff --git a/khal/utils.py b/khal/utils.py index ae144d86d..cc8206ec0 100644 --- a/khal/utils.py +++ b/khal/utils.py @@ -602,12 +602,14 @@ def split_ics(ics): :rtype list: """ cal = icalendar.Calendar.from_ical(ics) - vevents = [item for item in cal.walk() if item.name == 'VEVENT'] tzs = {item['TZID']: item for item in cal.walk() if item.name == 'VTIMEZONE'} events_grouped = defaultdict(list) - for event in vevents: - events_grouped[event['UID']].append(event) + for item in cal.walk(): + if item.name == 'VEVENT': + events_grouped[item['UID']].append(item) + else: + continue return [ics_from_list(events, tzs) for uid, events in sorted(events_grouped.items())] @@ -623,32 +625,44 @@ def ics_from_list(events, tzs): calendar.add('version', '2.0') calendar.add('prodid', '-//CALENDARSERVER.ORG//NONSGML Version 1//EN') - needed_tz = set() + needed_tz, missing_tz = set(), set() for sub_event in events: - # RRULE-UNTIL XXX can that even be anything but UTC? + # take care of RRULE-UNTIL XXX can that even be anything but UTC? # icalendar round-trip converts `TZID=a b` to `TZID="a b"` investigate, file bug XXX for prop in ['DTSTART', 'DTEND', 'DUE', 'EXDATE', 'RDATE', 'RECURRENCE-ID', 'DUE']: if isinstance(sub_event.get(prop), list): items = sub_event.get(prop) else: items = [sub_event.get(prop)] + for item in items: - try: - # if prop is a list, all items have the same parameters - if hasattr(item, 'dts'): - datetime_ = item.dts[0].dt - else: - datetime_ = item.dt - needed_tz.add(datetime_.tzinfo) - except AttributeError: + if not (hasattr(item, 'dt') or hasattr(item, 'dts')): continue + # if prop is a list, all items have the same parameters + datetime_ = item.dts[0].dt if hasattr(item, 'dts') else item.dt + + if not hasattr(datetime_, 'tzinfo'): + continue + + # check for datetimes' timezones which are not understood by + # icalendar + if datetime_.tzinfo is None and 'TZID' in item.params and \ + item.params['TZID'] not in missing_tz: + logger.warn( + 'Cannot find timezone `{}` in .ics file, using default timezone. ' + 'This can lead to erroneous time shifts'.format(item.params['TZID']) + ) + missing_tz.add(item.params['TZID']) + elif datetime_.tzinfo != pytz.UTC: + needed_tz.add(datetime_.tzinfo) for tzid in needed_tz: if str(tzid) in tzs: calendar.add_component(tzs[str(tzid)]) else: logger.warn( - 'Cannot find timezone `{}` in .ics file'.format(tzid)) # XXX + 'Cannot find timezone `{}` in .ics file, this could be a bug, ' + 'please report this issue at http://github.com/pimutils/khal/.'.format(tzid)) for sub_event in events: calendar.add_component(sub_event) return calendar.to_ical().decode('utf-8') diff --git a/tests/ics/part0.ics b/tests/ics/part0.ics index 1a449b0ab..b919d4819 100644 --- a/tests/ics/part0.ics +++ b/tests/ics/part0.ics @@ -2,93 +2,27 @@ BEGIN:VCALENDAR VERSION:2.0 PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN BEGIN:VTIMEZONE -TZID:IndianReunion -BEGIN:STANDARD -TZOFFSETFROM:+034152 -TZOFFSETTO:+0400 -TZNAME:RET -DTSTART:19110601T000000 -RDATE:19110601T000000 -END:STANDARD -END:VTIMEZONE -BEGIN:VTIMEZONE -TZID:Europe_Amsterdam +TZID:Europe_London BEGIN:DAYLIGHT -TZOFFSETFROM:+0100 -TZOFFSETTO:+0200 -TZNAME:CEST -DTSTART:19810329T020000 +TZOFFSETFROM:+0000 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19810329T010000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 END:DAYLIGHT BEGIN:STANDARD -TZOFFSETFROM:+0200 -TZOFFSETTO:+0100 -TZNAME:CET -DTSTART:19961027T030000 -RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 -END:STANDARD -END:VTIMEZONE -BEGIN:VTIMEZONE -TZID:Europe_Berlin -BEGIN:STANDARD -DTSTART;VALUE=DATE-TIME:20141026T020000 -TZNAME:CET -TZOFFSETFROM:+0200 -TZOFFSETTO:+0100 -RDATE:20151025T020000 -END:STANDARD -BEGIN:DAYLIGHT -DTSTART;VALUE=DATE-TIME:20140330T030000 -RDATE:20150329T030000,20160327T030000 -TZNAME:CEST TZOFFSETFROM:+0100 -TZOFFSETTO:+0200 -END:DAYLIGHT -END:VTIMEZONE -BEGIN:VTIMEZONE -TZID:America_New_York -BEGIN:STANDARD -DTSTART;VALUE=DATE-TIME:20141102T010000 -RDATE:20151101T010000 -TZNAME:EST -TZOFFSETFROM:-0400 -TZOFFSETTO:-0500 -END:STANDARD -BEGIN:DAYLIGHT -DTSTART;VALUE=DATE-TIME:20140309T030000 -RDATE:20150308T030000,20160313T030000 -TZNAME:EDT -TZOFFSETFROM:-0500 -TZOFFSETTO:-0400 -END:DAYLIGHT -END:VTIMEZONE -BEGIN:VTIMEZONE -TZID:America_Bogota -BEGIN:STANDARD -TZOFFSETFROM:-0400 -TZOFFSETTO:-0500 -TZNAME:COT -DTSTART:19930404T000000 -RDATE:19930404T000000 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:19961027T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 END:STANDARD END:VTIMEZONE BEGIN:VEVENT SUMMARY:An Event -DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 -DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 -RDATE;TZID=IndianReunion:20140418T113000 -RDATE;TZID=America_Bogota:20140411T113000,20140413T113000 -RDATE;TZID=America_Bogota:20140415T113000 -RRULE:FREQ=MONTHLY;COUNT=6 -DTSTAMP;VALUE=DATE-TIME:20140401T234817Z -UID:abcde -END:VEVENT -BEGIN:VEVENT -SUMMARY:An Updated Event -DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 -DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 +DTSTART;TZID=Europe_London;VALUE=DATE-TIME:20140509T193000 +DTEND;TZID=Europe_London;VALUE=DATE-TIME:20140509T203000 DTSTAMP;VALUE=DATE-TIME:20140401T234817Z -UID:abcde -RECURRENCE-ID;TZID=Europe_Amsterdam:20140707T070000 +UID:123 END:VEVENT END:VCALENDAR diff --git a/tests/ics/part1.ics b/tests/ics/part1.ics index b919d4819..1a449b0ab 100644 --- a/tests/ics/part1.ics +++ b/tests/ics/part1.ics @@ -2,27 +2,93 @@ BEGIN:VCALENDAR VERSION:2.0 PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN BEGIN:VTIMEZONE -TZID:Europe_London +TZID:IndianReunion +BEGIN:STANDARD +TZOFFSETFROM:+034152 +TZOFFSETTO:+0400 +TZNAME:RET +DTSTART:19110601T000000 +RDATE:19110601T000000 +END:STANDARD +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Europe_Amsterdam BEGIN:DAYLIGHT -TZOFFSETFROM:+0000 -TZOFFSETTO:+0100 -TZNAME:BST -DTSTART:19810329T010000 +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19810329T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 END:DAYLIGHT BEGIN:STANDARD -TZOFFSETFROM:+0100 -TZOFFSETTO:+0000 -TZNAME:GMT -DTSTART:19961027T020000 +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19961027T030000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 END:STANDARD END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:Europe_Berlin +BEGIN:STANDARD +DTSTART;VALUE=DATE-TIME:20141026T020000 +TZNAME:CET +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +RDATE:20151025T020000 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART;VALUE=DATE-TIME:20140330T030000 +RDATE:20150329T030000,20160327T030000 +TZNAME:CEST +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:America_New_York +BEGIN:STANDARD +DTSTART;VALUE=DATE-TIME:20141102T010000 +RDATE:20151101T010000 +TZNAME:EST +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART;VALUE=DATE-TIME:20140309T030000 +RDATE:20150308T030000,20160313T030000 +TZNAME:EDT +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VTIMEZONE +TZID:America_Bogota +BEGIN:STANDARD +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +TZNAME:COT +DTSTART:19930404T000000 +RDATE:19930404T000000 +END:STANDARD +END:VTIMEZONE BEGIN:VEVENT SUMMARY:An Event -DTSTART;TZID=Europe_London;VALUE=DATE-TIME:20140509T193000 -DTEND;TZID=Europe_London;VALUE=DATE-TIME:20140509T203000 +DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 +DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 +RDATE;TZID=IndianReunion:20140418T113000 +RDATE;TZID=America_Bogota:20140411T113000,20140413T113000 +RDATE;TZID=America_Bogota:20140415T113000 +RRULE:FREQ=MONTHLY;COUNT=6 +DTSTAMP;VALUE=DATE-TIME:20140401T234817Z +UID:abcde +END:VEVENT +BEGIN:VEVENT +SUMMARY:An Updated Event +DTSTART;TZID=Europe_Berlin;VALUE=DATE-TIME:20140409T093000 +DTEND;TZID=America_New_York;VALUE=DATE-TIME:20140409T103000 DTSTAMP;VALUE=DATE-TIME:20140401T234817Z -UID:123 +UID:abcde +RECURRENCE-ID;TZID=Europe_Amsterdam:20140707T070000 END:VEVENT END:VCALENDAR From 59f42b082daf499b30ba92d1a1055d7fd94d644f Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Tue, 4 Oct 2016 17:33:30 +0200 Subject: [PATCH 05/13] Better warning on invalid timezones. --- khal/khalendar/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/khal/khalendar/utils.py b/khal/khalendar/utils.py index 66f3f4e1f..120bfacbd 100644 --- a/khal/khalendar/utils.py +++ b/khal/khalendar/utils.py @@ -141,11 +141,14 @@ def sanitize(vevent, default_timezone, href='', calendar=''): # RRULE:UNTIL) for prop in ['DTSTART', 'DTEND', 'DUE', 'RECURRENCE-ID']: if prop in vevent and invalid_timezone(vevent[prop]): + timezone = vevent[prop].params.get('TZID') value = default_timezone.localize(vevent.pop(prop).dt) vevent.add(prop, value) logger.warn( - '{} has invalid or incomprehensible timezone ' - 'information in {} in {}'.format(prop, href, calendar)) + "{} localized in invalid or incomprehensible timezone `{}` in {}/{}. " + "This could lead to this event being wronlgy displayed." + "".format(prop, timezone, calendar, href) + ) vdtstart = vevent.pop('DTSTART', None) vdtend = vevent.pop('DTEND', None) From 8f66019b1a8f6198000808d25d3e1ab9b0c676e3 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Tue, 4 Oct 2016 17:34:48 +0200 Subject: [PATCH 06/13] RRULE:UNTIL must be in UTC (if localized). --- khal/khalendar/utils.py | 3 +-- khal/utils.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/khal/khalendar/utils.py b/khal/khalendar/utils.py index 120bfacbd..6f5032702 100644 --- a/khal/khalendar/utils.py +++ b/khal/khalendar/utils.py @@ -137,8 +137,7 @@ def sanitize(vevent, default_timezone, href='', calendar=''): """ # convert localized datetimes with timezone information we don't # understand to the default timezone - # TODO do this for everything where a TZID can appear (RDATE, EXDATE, - # RRULE:UNTIL) + # TODO do this for everything where a TZID can appear (RDATE, EXDATE) for prop in ['DTSTART', 'DTEND', 'DUE', 'RECURRENCE-ID']: if prop in vevent and invalid_timezone(vevent[prop]): timezone = vevent[prop].params.get('TZID') diff --git a/khal/utils.py b/khal/utils.py index cc8206ec0..b34e04980 100644 --- a/khal/utils.py +++ b/khal/utils.py @@ -627,7 +627,6 @@ def ics_from_list(events, tzs): needed_tz, missing_tz = set(), set() for sub_event in events: - # take care of RRULE-UNTIL XXX can that even be anything but UTC? # icalendar round-trip converts `TZID=a b` to `TZID="a b"` investigate, file bug XXX for prop in ['DTSTART', 'DTEND', 'DUE', 'EXDATE', 'RDATE', 'RECURRENCE-ID', 'DUE']: if isinstance(sub_event.get(prop), list): From 43a538827212e23a077e3dcce444201be2478cb1 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Tue, 4 Oct 2016 17:40:03 +0200 Subject: [PATCH 07/13] 084 release note drafts --- doc/source/news/khal084.rst | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/source/news/khal084.rst diff --git a/doc/source/news/khal084.rst b/doc/source/news/khal084.rst new file mode 100644 index 000000000..656d31adb --- /dev/null +++ b/doc/source/news/khal084.rst @@ -0,0 +1,45 @@ +khal v0.8.4 released +==================== + +.. feed-entry:: + :date: 2016-10-XX + +`khal v0.8.4`_ (pypi_) is a bugfix release that fixes a critical bug in `khal +import`. All users are advised to upgrade as soon as possible. + +Details +~~~~~~~ +If importing events from `.ics` files, any VTIMEZONEs (specifications of the +timezone) would *not* be imported with those events. +As khal understands Olson DB timezone specifiers (such as "Europe/Berlin" or +"America/New_York", events using those timezones are displayed in the correct +timezone, but all other events are displayed as if they were in the configured +*default timezone*. +**This can lead to imported events being shown at wrong times!** + + +Solution +~~~~~~~~ +First, please upgrade khal to either v0.8.4 or, if you are using a version of khal directly +from the git repository, upgrade to the latest version from github_. + +To see if you are affected by this bug, delete your local khal caching db, +(usually `~/.local/share/khal/khal.db`), re-run khal and watch out for lines +looking like this: +``warning: $PROPERTY has invalid or incomprehensible timezone information in +$long_uid.ics in $my_collection``. +You will then need to edit these files by hand and either replace the timezone +identifiers with the corresponding one from the Olson DB (e.g., change +`Europe_Berlin` to `Europe/Berlin`) or copy original VTIMZONE definition in. + +If you have any problems with this, please either open an `issue at github`_ or come into +our `irc channel`_ (`#pimutils` on Freenode). + +We are sorry for any inconveniences this is causing you! + + +.. _khal v0.8.4: https://lostpackets.de/khal/downloads/khal-0.8.4.tar.gz +.. _github: https://github.com/pimutils/khal/ +.. _issue at github: https://github.com/pimutils/khal/issues +.. _pypi: https://pypi.python.org/pypi/khal/ +.. _irc channel: irc://#pimutils@Freenode From 67bcf1e86caf47ff3b8087f89d56b258e4d11311 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Tue, 4 Oct 2016 18:22:27 +0200 Subject: [PATCH 08/13] Copyright header for khal/khalendar/utils.py --- khal/khalendar/utils.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/khal/khalendar/utils.py b/khal/khalendar/utils.py index 6f5032702..9b74244b3 100644 --- a/khal/khalendar/utils.py +++ b/khal/khalendar/utils.py @@ -1,4 +1,25 @@ - +# Copyright (c) 2013-2016 Christian Geier et al. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""collection of utility functions""" from datetime import datetime, timedelta import calendar From 74434bd4364bcd5a8e33da2482e6da9e7160e3c3 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Tue, 4 Oct 2016 18:56:13 +0200 Subject: [PATCH 09/13] flake8ing --- khal/controllers.py | 3 +-- tests/utils_test.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/khal/controllers.py b/khal/controllers.py index bc1c896d1..8f33b65eb 100644 --- a/khal/controllers.py +++ b/khal/controllers.py @@ -27,7 +27,7 @@ import pytz -from collections import defaultdict, OrderedDict +from collections import OrderedDict from shutil import get_terminal_size from datetime import timedelta, datetime @@ -39,7 +39,6 @@ from khal.khalendar.exceptions import ReadOnlyCalendarError, DuplicateUid from khal.exceptions import InvalidDate, FatalError from khal.khalendar.event import Event -from khal.khalendar.backend import sort_key from khal import __version__, __productname__ from khal.log import logger from .terminal import merge_columns diff --git a/tests/utils_test.py b/tests/utils_test.py index 38baf6de7..14d25bea0 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -3,7 +3,6 @@ from collections import OrderedDict import textwrap -import icalendar import pytz from freezegun import freeze_time @@ -12,8 +11,7 @@ from khal import utils import pytest -from .utils import _get_all_vevents_file, _get_text, \ - normalize_component +from .utils import _get_text, normalize_component today = date.today() From 8ca650d17b2c9dbb23e9e6f7fc93f16a1b8c901a Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Tue, 4 Oct 2016 18:56:22 +0200 Subject: [PATCH 10/13] typo --- khal/khalendar/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/khal/khalendar/utils.py b/khal/khalendar/utils.py index 9b74244b3..93b2734d2 100644 --- a/khal/khalendar/utils.py +++ b/khal/khalendar/utils.py @@ -166,7 +166,7 @@ def sanitize(vevent, default_timezone, href='', calendar=''): vevent.add(prop, value) logger.warn( "{} localized in invalid or incomprehensible timezone `{}` in {}/{}. " - "This could lead to this event being wronlgy displayed." + "This could lead to this event being wrongly displayed." "".format(prop, timezone, calendar, href) ) From 83be6b556c113617fd61c73a35992fd86bb20a96 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Thu, 6 Oct 2016 17:58:06 +0200 Subject: [PATCH 11/13] Support for random uids in import --- khal/controllers.py | 15 +++++++-------- khal/utils.py | 16 +++++++++++++--- tests/utils_test.py | 43 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/khal/controllers.py b/khal/controllers.py index 8f33b65eb..35b3d2e76 100644 --- a/khal/controllers.py +++ b/khal/controllers.py @@ -526,26 +526,25 @@ def import_ics(collection, conf, ics, batch=False, random_uid=False, format=None :param batch: setting this to True will insert without asking for approval, even when an event with the same uid already exists :type batch: bool + :param random_uid: whether to assign a random UID to imported events or not + :type random_uid: bool + :param format: the format string to print events with + :type format: str """ if format is None: format = conf['view']['event_format'] - - vevents = utils.split_ics(ics) - + vevents = utils.split_ics(ics, random_uid) for vevent in vevents: - import_event( - vevent, collection, conf['locale'], batch, random_uid, format, env, - ) + import_event(vevent, collection, conf['locale'], batch, format, env) -def import_event(vevent, collection, locale, batch, random_uid, format=None, env=None): +def import_event(vevent, collection, locale, batch, format=None, env=None): """import one event into collection, let user choose the collection :type vevent: list of vevents, which can be more than one VEVENT, i.e., the same UID, i.e., one "master" event and (optionally) 1+ RECURRENCE-ID events :type vevent: list(str) """ - # TODO re-enable random_uid # print all sub-events if not batch: for item in icalendar.Calendar.from_ical(vevent).walk(): diff --git a/khal/utils.py b/khal/utils.py index b34e04980..65723c1af 100644 --- a/khal/utils.py +++ b/khal/utils.py @@ -593,12 +593,14 @@ def new_event(locale, dtstart=None, dtend=None, summary=None, timezone=None, return event -def split_ics(ics): +def split_ics(ics, random_uid=False): """split an ics string into several according to VEVENT's UIDs and sort the right VTIMEZONEs accordingly ignores all other ics components :type ics: str + :param random_uid: assign random uids to all events + :type random_uid: bool :rtype list: """ cal = icalendar.Calendar.from_ical(ics) @@ -610,14 +612,17 @@ def split_ics(ics): events_grouped[item['UID']].append(item) else: continue - return [ics_from_list(events, tzs) for uid, events in sorted(events_grouped.items())] + return [ics_from_list(events, tzs, random_uid) for uid, events in + sorted(events_grouped.items())] -def ics_from_list(events, tzs): +def ics_from_list(events, tzs, random_uid=False): """convert an iterable of icalendar.Events to an icalendar.Calendar :params events: list of events all with the same uid :type events: list(icalendar.cal.Event) + :param random_uid: assign random uids to all events + :type random_uid: bool :param tzs: collection of timezones :type tzs: dict(icalendar.cal.Vtimzone """ @@ -625,8 +630,13 @@ def ics_from_list(events, tzs): calendar.add('version', '2.0') calendar.add('prodid', '-//CALENDARSERVER.ORG//NONSGML Version 1//EN') + if random_uid: + new_uid = generate_random_uid() + needed_tz, missing_tz = set(), set() for sub_event in events: + if random_uid: + sub_event['UID'] = new_uid # icalendar round-trip converts `TZID=a b` to `TZID="a b"` investigate, file bug XXX for prop in ['DTSTART', 'DTEND', 'DUE', 'EXDATE', 'RDATE', 'RECURRENCE-ID', 'DUE']: if isinstance(sub_event.get(prop), list): diff --git a/tests/utils_test.py b/tests/utils_test.py index 14d25bea0..c623015e5 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -2,7 +2,9 @@ from datetime import date, datetime, time, timedelta from collections import OrderedDict import textwrap +import random +import icalendar import pytz from freezegun import freeze_time @@ -80,6 +82,11 @@ def _replace_uid(event): return event +def _get_TZIDs(lines): + """from a list of strings, get all unique strings that start with TZID""" + return sorted((line for line in lines if line.startswith('TZID'))) + + def test_normalize_component(): assert normalize_component(textwrap.dedent(""" BEGIN:VEVENT @@ -513,13 +520,37 @@ def test_split_ics(): vevents0 = vevents[0].split('\r\n') vevents1 = vevents[1].split('\r\n') - part0 = _get_text('part1').split('\n') - part1 = _get_text('part0').split('\n') + part0 = _get_text('part0').split('\n') + part1 = _get_text('part1').split('\n') + + assert _get_TZIDs(vevents0) == _get_TZIDs(part0) + assert _get_TZIDs(vevents1) == _get_TZIDs(part1) + + assert sorted(vevents0) == sorted(part0) + assert sorted(vevents1) == sorted(part1) + + +def test_split_ics_random_uid(): + random.seed(123) + cal = _get_text('cal_lots_of_timezones') + vevents = utils.split_ics(cal, random_uid=True) + + part0 = _get_text('part0').split('\n') + part1 = _get_text('part1').split('\n') + + for item in icalendar.Calendar.from_ical(vevents[0]).walk(): + if item.name == 'VEVENT': + assert item['UID'] == 'DRF0RGCY89VVDKIV9VPKA1FYEAU2GCFJIBS1' + for item in icalendar.Calendar.from_ical(vevents[1]).walk(): + if item.name == 'VEVENT': + assert item['UID'] == '4Q4CTV74N7UAZ618570X6CLF5QKVV9ZE3YVB' + + # after replacing the UIDs, everything should be as above + vevents0 = vevents[0].replace('DRF0RGCY89VVDKIV9VPKA1FYEAU2GCFJIBS1', '123').split('\r\n') + vevents1 = vevents[1].replace('4Q4CTV74N7UAZ618570X6CLF5QKVV9ZE3YVB', 'abcde').split('\r\n') - assert sorted([line for line in vevents1 if line.startswith('TZID')]) == \ - sorted([line for line in part1 if line.startswith('TZID')]) - assert sorted([line for line in vevents0 if line.startswith('TZID')]) == \ - sorted([line for line in part0 if line.startswith('TZID')]) + assert _get_TZIDs(vevents0) == _get_TZIDs(part0) + assert _get_TZIDs(vevents1) == _get_TZIDs(part1) assert sorted(vevents0) == sorted(part0) assert sorted(vevents1) == sorted(part1) From 73852395d94bbbda1d86d1355455e4cd45d6c446 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Thu, 6 Oct 2016 18:31:14 +0200 Subject: [PATCH 12/13] Updated changelog --- CHANGELOG.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ac265c08d..7b94560d8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,17 @@ ikhal to another date (while the event column is in focus), that date should be highlighted in the calendar (Christian Geier) +0.8.4 +===== +released 2016-10-06 + +* **IMPORTANT BUGFIX** fixed a bug that lead to imported events being + erroneously shifted if they had a timezone identifier that wasn't an Olson + database identifier. All users are advised to upgrade as soon as possible. To + see if you are affected by this and how to resolve any issues, please see the + release announcement (khal/doc/source/news/khal084.rst or + http://lostpackets.de/khal/news/khal084.html). + 0.8.3 ===== released 2016-08-28 From 7b360dcac0a2f398ab52081edc80cad2df058f61 Mon Sep 17 00:00:00 2001 From: Christian Geier Date: Thu, 6 Oct 2016 18:39:39 +0200 Subject: [PATCH 13/13] Updated release notes --- CHANGELOG.rst | 3 ++- doc/source/news.rst | 1 + doc/source/news/khal084.rst | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7b94560d8..1d7b75241 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -54,7 +54,8 @@ released 2016-10-06 database identifier. All users are advised to upgrade as soon as possible. To see if you are affected by this and how to resolve any issues, please see the release announcement (khal/doc/source/news/khal084.rst or - http://lostpackets.de/khal/news/khal084.html). + http://lostpackets.de/khal/news/khal084.html). Thanks to Wayne Werner for + finding and reporting this bug. 0.8.3 ===== diff --git a/doc/source/news.rst b/doc/source/news.rst index 194056c9c..ea7ec3f45 100644 --- a/doc/source/news.rst +++ b/doc/source/news.rst @@ -12,6 +12,7 @@ available as an `rss feed `_ |rss|. :title: khal news :link: http://lostpackets.de/khal/ + news/khal084 news/khal083 news/khal082 news/khal081 diff --git a/doc/source/news/khal084.rst b/doc/source/news/khal084.rst index 656d31adb..25c3c681c 100644 --- a/doc/source/news/khal084.rst +++ b/doc/source/news/khal084.rst @@ -2,10 +2,10 @@ khal v0.8.4 released ==================== .. feed-entry:: - :date: 2016-10-XX + :date: 2016-10-06 -`khal v0.8.4`_ (pypi_) is a bugfix release that fixes a critical bug in `khal -import`. All users are advised to upgrade as soon as possible. +`khal v0.8.4`_ (pypi_) is a bugfix release that fixes a **critical bug** in `khal +import`. **All users are advised to upgrade as soon as possible**. Details ~~~~~~~