diff --git a/Makefile b/Makefile index 3914c67..35c358a 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,8 @@ logs: docker-compose logs -f --tail=500 backup: - docker-compose exec web ./manage.py dumpdata --natural-foreign --indent=4 -e contenttypes -e auth.Permission -e sessions -e wagtailcore.pagerevision -e wagtailcore.groupcollectionpermission > ~/publichealth.home.json + docker-compose start postgres + docker-compose exec web ./manage.py dumpdata --natural-foreign -e auth.permission -e contenttypes -e wagtailcore.GroupCollectionPermission -e wagtailimages.rendition -e sessions -e feedler.feedlysettings > ~/publichealth.home.json zip ~/publichealth.home.json.`date +"%d%m%Y-%H%M"`.zip ~/publichealth.home.json rm ~/publichealth.home.json diff --git a/feedler/admin.py b/feedler/admin.py new file mode 100644 index 0000000..391117d --- /dev/null +++ b/feedler/admin.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +from django.db import models +from django.contrib.auth.decorators import login_required +from django.utils.decorators import method_decorator +from django.conf.urls import url +from django.core.urlresolvers import reverse +from django.utils.functional import cached_property +from django.utils.translation import ugettext as _ +from django.shortcuts import redirect + +from wagtail.contrib.modeladmin.helpers import AdminURLHelper, ButtonHelper +from wagtail.contrib.modeladmin.options import ModelAdmin +from wagtail.contrib.modeladmin.views import IndexView +from wagtail.wagtailadmin import messages + +from feedler.models import Entry +from feedler.refresh import refresh_streams +from feedler.models.admin import FeedlySettings + +class RefreshButtonHelper(ButtonHelper): + """ + This helper constructs a refresh button + """ + button_classnames = ['icon', 'icon-download'] + def refresh_button(self, classnames_add=None, classnames_exclude=None): + if classnames_add is None: classnames_add = [] + if classnames_exclude is None: classnames_exclude = [] + classnames = self.button_classnames + classnames_add + cn = self.finalise_classname(classnames, classnames_exclude) + text = _('Sync {}'.format(self.verbose_name_plural.title())) + return { + 'url': self.url_helper.get_action_url('refresh', query_params=self.request.GET), + 'label': text, 'classname': cn, 'title': text, + } + +class RefreshAdminURLHelper(AdminURLHelper): + """ + This helper constructs the different urls, to overwrite the default behaviour + and append the filters to the action. + """ + non_object_specific_actions = ('create', 'choose_parent', 'index', 'refresh') + def get_action_url(self, action, *args, **kwargs): + query_params = kwargs.pop('query_params', None) + url_name = self.get_action_url_name(action) + if action in self.non_object_specific_actions: + url = reverse(url_name) + else: + url = reverse(url_name, args=args, kwargs=kwargs) + if query_params: + url += '?{params}'.format(params=query_params.urlencode()) + return url + def get_action_url_pattern(self, action): + if action in self.non_object_specific_actions: + return self._get_action_url_pattern(action) + return self._get_object_specific_action_url_pattern(action) + +class RefreshView(IndexView): + """ + A Class Based View which will handle the button click + """ + # def export_csv(self): + # data = self.queryset.all() + # response = ... + # return response + @method_decorator(login_required) + def dispatch(self, request, *args, **kwargs): + super().dispatch(request, *args, **kwargs) + if not refresh_streams(FeedlySettings.for_site(request.site)): + messages.error( + request, _('Sorry, could not refresh streams. Please contact your administrator.')) + return redirect('/admin/feedler/entry/') + + +class EntryModelAdminMixin(object): + """ + A mixin to add to your model admin which hooks the different helpers, the view + and register the new urls. + """ + button_helper_class = RefreshButtonHelper + url_helper_class = RefreshAdminURLHelper + view_class = RefreshView + + def get_admin_urls_for_registration(self): + urls = super().get_admin_urls_for_registration() + urls += ( + url( + self.url_helper.get_action_url_pattern('refresh'), + self.refresh_view, + name=self.url_helper.get_action_url_name('refresh') + ), + ) + return urls + + def refresh_view(self, request): + kwargs = {'model_admin': self} + view_class = self.view_class + return view_class.as_view(**kwargs)(request) diff --git a/feedler/migrations/0007_auto_20171016_1059.py b/feedler/migrations/0007_auto_20171016_1059.py new file mode 100644 index 0000000..fa988ac --- /dev/null +++ b/feedler/migrations/0007_auto_20171016_1059.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2017-10-16 08:59 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('feedler', '0006_auto_20171012_1458'), + ] + + operations = [ + migrations.RenameField( + model_name='feedlysettings', + old_name='feedly_stream', + new_name='streams', + ), + migrations.RemoveField( + model_name='feedlysettings', + name='feedly_auth', + ), + migrations.RemoveField( + model_name='feedlysettings', + name='feedly_pages', + ), + migrations.AddField( + model_name='feedlysettings', + name='refresh', + field=models.CharField(blank=True, help_text='Refresh Token for automatic update (pro account)', max_length=1024), + ), + migrations.AddField( + model_name='feedlysettings', + name='token', + field=models.CharField(blank=True, help_text='Access Token from feedly.com/v3/auth/dev', max_length=1024), + ), + migrations.AlterField( + model_name='stream', + name='ident', + field=models.CharField(help_text='Example: enterprise/myuser/tag/abcd...', max_length=255), + ), + ] diff --git a/feedler/models/admin.py b/feedler/models/admin.py index b259777..275ba31 100644 --- a/feedler/models/admin.py +++ b/feedler/models/admin.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -import requests, json, codecs - from django.contrib import admin from django.db import models @@ -11,58 +9,21 @@ from django.core.mail import send_mail from wagtail.contrib.settings.models import BaseSetting, register_setting -from .models import Entry, Stream -import feedler.feedparser as feedparser +from .models import Stream -import logging -logger = logging.getLogger('feedler') - -# Feedly integration module +from feedler.refresh import refresh_streams @register_setting class FeedlySettings(BaseSetting): - feedly_auth = models.TextField( - help_text='Your developer authorization key', blank=True) - feedly_pages = models.IntegerField( - choices=( - (1, '2'), - (2, '5'), - (3, '10'), - (4, '50'), - ), blank=True, null=True, editable=False, - help_text='How many pages to fetch?' - ) - feedly_stream = models.ManyToManyField(Stream, + streams = models.ManyToManyField(Stream, help_text='Which streams to update') + token = models.CharField(max_length=1024, blank=True, + help_text='Access Token from feedly.com/v3/auth/dev') + refresh = models.CharField(max_length=1024, blank=True, + help_text='Refresh Token for automatic update (pro account)') class Meta: verbose_name = 'Feedly' -API_BASEURL = 'https://cloud.feedly.com/v3/streams/contents?streamId=' - -@receiver(pre_save, sender=FeedlySettings) -def handle_save_settings(sender, instance, *args, **kwargs): - if instance.feedly_auth: - streams = instance.feedly_stream.all() - for stream in streams: - # Start a request to download the feed - logger.info("Processing stream %s" % stream.title) - url = API_BASEURL + stream.ident - headers = { - 'Authorization': 'OAuth ' + instance.feedly_auth - } - contents = requests.get(url, headers=headers).json() - if 'errorMessage' in contents: - raise PermissionError(contents['errorMessage']) - for raw_entry in contents['items']: - eid = raw_entry['id'] - # Create or update data - try: - entry = Entry.objects.get(entry_id=eid) - logger.info("Updating entry '%s'" % eid) - except Entry.DoesNotExist: - logger.info("Adding entry '%s'" % eid) - entry = Entry() - # Parse the Feedly object - entry = feedparser.parse(entry, raw_entry, stream) - # Persist resulting object - entry.save() +# @receiver(pre_save, sender=FeedlySettings) +# def handle_save_settings(sender, instance, *args, **kwargs): +# if instance.token: refresh_streams(instance) diff --git a/feedler/models/models.py b/feedler/models/models.py index a1f9a82..ae79926 100644 --- a/feedler/models/models.py +++ b/feedler/models/models.py @@ -10,7 +10,8 @@ from wagtail.wagtailcore.fields import RichTextField class Stream(models.Model): title = models.CharField(max_length=255) - ident = models.CharField(max_length=255) + ident = models.CharField(max_length=255, + help_text='Example: enterprise/myuser/tag/abcd...') def __str__(self): return self.title diff --git a/feedler/refresh.py b/feedler/refresh.py new file mode 100644 index 0000000..f5cc3df --- /dev/null +++ b/feedler/refresh.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +import requests, json, codecs + +import logging +logger = logging.getLogger('feedler') + +from .models import Entry +from feedler import feedparser + +API_BASEURL = 'https://cloud.feedly.com/v3/' +API_STREAMS = API_BASEURL + 'streams/contents?streamId=' +API_TOKENS = API_BASEURL + 'auth/token' + +def refresh_streams(settings): + # Iterate through all saved streams + logger.debug("Refreshing all streams") + for stream in settings.streams.all(): + if not refresh_stream(stream, settings): + return False + return True + +def refresh_token(settings): + # Request a new token + url = API_TOKENS + if not settings.refresh: + logger.warn("No Refresh token available") + return False + logger.info("Refreshing Feedly access token") + payload = { + 'refresh_token': settings.refresh, + 'client_id': 'feedlydev', + 'client_secret': 'feedlydev', + 'grant_type': 'refresh_token' + } + contents = requests.post(url, data=payload).json() + if not 'access_token' in contents or not contents['access_token']: + logger.warn("Access token could not be refreshed.") + logger.debug(contents) + return False + settings.token = contents['access_token'] + settings.save() + return True + +def refresh_stream(stream, settings, retry=False): + # Start a request to download the feed for a particular stream + logger.info("Processing stream %s" % stream.title) + url = API_STREAMS + stream.ident + headers = { 'Authorization': 'OAuth ' + settings.token } + contents = requests.get(url, headers=headers).json() + if 'errorMessage' in contents: + # Usually this is a token expired + if 'token expired' in contents['errorMessage'] or 'unauthorized' in contents['errorMessage']: + logger.debug(contents['errorMessage']) + if not refresh_token(settings): return False + # Make another attempt + if retry or not refresh_stream(stream, settings, True): + return False + else: + logger.error(contents['errorMessage']) + return False + for raw_entry in contents['items']: + eid = raw_entry['id'] + # Create or update data + try: + entry = Entry.objects.get(entry_id=eid) + logger.info("Skipping entry '%s'" % eid) + except Entry.DoesNotExist: + logger.info("Adding entry '%s'" % eid) + entry = Entry() + # Parse the Feedly object + entry = feedparser.parse(entry, raw_entry, stream) + # Persist resulting object + entry.save() + return True diff --git a/feedler/templates/modeladmin/feedler/entry/index.html b/feedler/templates/modeladmin/feedler/entry/index.html new file mode 100644 index 0000000..234e82f --- /dev/null +++ b/feedler/templates/modeladmin/feedler/entry/index.html @@ -0,0 +1,12 @@ +{% extends "modeladmin/index.html" %} + +{% block header_extra %} + {% if user_can_create %} +
+
+ {% include 'modeladmin/includes/button.html' with button=view.button_helper.refresh_button %} +
+
+ {% endif %} + {{ block.super }}{% comment %}Show original buttons{% endcomment %} +{% endblock %} diff --git a/feedler/wagtail_hooks.py b/feedler/wagtail_hooks.py index 971c2c7..0129417 100644 --- a/feedler/wagtail_hooks.py +++ b/feedler/wagtail_hooks.py @@ -3,9 +3,10 @@ from wagtail.contrib.modeladmin.options import ( ModelAdmin, modeladmin_register) +from .admin import EntryModelAdminMixin from .models import Entry, Stream -class EntryModelAdmin(ModelAdmin): +class EntryModelAdmin(EntryModelAdminMixin, ModelAdmin): model = Entry menu_icon = 'date' menu_order = 200 @@ -20,7 +21,7 @@ modeladmin_register(EntryModelAdmin) class StreamModelAdmin(ModelAdmin): model = Stream menu_icon = 'date' - menu_order = 1000 + menu_order = 900 add_to_settings_menu = True exclude_from_explorer = True list_display = ('title', 'ident') diff --git a/publichealth/home/migrations/0021_auto_20171013_2321.py b/publichealth/home/migrations/0021_auto_20171013_2321.py new file mode 100644 index 0000000..37f3aaa --- /dev/null +++ b/publichealth/home/migrations/0021_auto_20171013_2321.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2017-10-13 21:21 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('home', '0020_auto_20170920_1204'), + ] + + operations = [ + migrations.AlterModelOptions( + name='dataletssettings', + options={'verbose_name': 'Datalets'}, + ), + migrations.AlterField( + model_name='dataletssettings', + name='feedback_comment', + field=models.TextField(blank=True, help_text='Any questions or general feedback', verbose_name='Comments..'), + ), + migrations.AlterField( + model_name='dataletssettings', + name='feedback_question', + field=models.TextField(blank=True, help_text='Who should we reply to for questions?', verbose_name='Name'), + ), + migrations.AlterField( + model_name='dataletssettings', + name='feedback_status', + field=models.IntegerField(blank=True, choices=[(1, '★'), (2, '★★'), (3, '★★★'), (4, '★★★★'), (5, '★★★★★')], help_text='How are you enjoying Wagtail?', null=True, verbose_name='Rating'), + ), + ] diff --git a/publichealth/home/models/admin.py b/publichealth/home/models/admin.py index 8cf4eec..e250a41 100644 --- a/publichealth/home/models/admin.py +++ b/publichealth/home/models/admin.py @@ -9,23 +9,24 @@ from wagtail.contrib.settings.models import BaseSetting, register_setting # A simple feedback module built into the site admin -@register_setting +@register_setting(icon='help') class DataletsSettings(BaseSetting): - feedback_question = models.TextField( - help_text='Send us a question', blank=True) - feedback_status = models.IntegerField( + feedback_question = models.TextField(verbose_name='Name', + help_text='Who should we reply to for questions?', blank=True) + feedback_status = models.IntegerField(verbose_name='Rating', choices=( - (1, ':-('), - (2, ':-|'), - (3, ':-)'), - (4, ':-D'), + (1, u'★'), + (2, u'★'*2), + (3, u'★'*3), + (4, u'★'*4), + (5, u'★'*5), ), blank=True, null=True, help_text='How are you enjoying Wagtail?' ) - feedback_comment = models.TextField( - help_text='Any general feedback', blank=True) + feedback_comment = models.TextField(verbose_name='Comments..', + help_text='Any questions or general feedback', blank=True) class Meta: - verbose_name = 'Get support' + verbose_name = 'Datalets' @receiver(pre_save, sender=DataletsSettings) def handle_save_settings(sender, instance, *args, **kwargs): @@ -41,3 +42,4 @@ def handle_save_settings(sender, instance, *args, **kwargs): instance.feedback_status = None instance.feedback_question = "" instance.feedback_comment = "" + instance.save() diff --git a/publichealth/settings/dev.py b/publichealth/settings/dev.py index a22cce2..e855f5d 100644 --- a/publichealth/settings/dev.py +++ b/publichealth/settings/dev.py @@ -21,6 +21,10 @@ DATABASES = { } } +INSTALLED_APPS = INSTALLED_APPS + [ + 'wagtail.contrib.wagtailstyleguide', +] + try: from .local import * except ImportError: diff --git a/publichealth/settings/production.py b/publichealth/settings/production.py index eb23ad7..7b753de 100644 --- a/publichealth/settings/production.py +++ b/publichealth/settings/production.py @@ -156,6 +156,12 @@ LOGGING = { } }, 'loggers': { + 'feedler': { + 'handlers': [], + 'level': 'INFO', + 'propagate': False, + 'formatter': 'verbose', + }, 'publichealth': { 'handlers': [], 'level': 'INFO', diff --git a/requirements-test.txt b/requirements-test.txt index be7e033..310cf26 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ -pytest==3.0.3 -pytest-splinter==1.7.6 -tox==2.3.1 +pytest>=3.0.3 +pytest-splinter>=1.8.5 +tox>=2.9.1 +django-debug-toolbar>=1.8