ungleich-public
/
public-health-ch
6
1
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

Merge pull request #20 from loleg/refresh-token 

Refresh token and sync button
This commit is contained in:
datalets 2017-10-16 12:03:51 +02:00 committed by GitHub
parent da80ab7d7e 8fe9b6812a
commit 5fbdd3da06
13 changed files with 306 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -66,7 +66,8 @@ logs:
docker-compose logs -f --tail=500
backup:
docker-compose exec web ./manage.py dumpdata --natural-foreign --indent=4 -e contenttypes -e auth.Permission -e sessions -e wagtailcore.pagerevision -e wagtailcore.groupcollectionpermission > ~/publichealth.home.json
docker-compose start postgres
docker-compose exec web ./manage.py dumpdata --natural-foreign -e auth.permission -e contenttypes -e wagtailcore.GroupCollectionPermission -e wagtailimages.rendition -e sessions -e feedler.feedlysettings > ~/publichealth.home.json
zip ~/publichealth.home.json.`date +"%d%m%Y-%H%M"`.zip ~/publichealth.home.json
rm ~/publichealth.home.json

98
feedler/admin.py Normal file
View File

@ -0,0 +1,98 @@
# -*- coding: utf-8 -*-
from django.db import models
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from django.conf.urls import url
from django.core.urlresolvers import reverse
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from django.shortcuts import redirect
from wagtail.contrib.modeladmin.helpers import AdminURLHelper, ButtonHelper
from wagtail.contrib.modeladmin.options import ModelAdmin
from wagtail.contrib.modeladmin.views import IndexView
from wagtail.wagtailadmin import messages
from feedler.models import Entry
from feedler.refresh import refresh_streams
from feedler.models.admin import FeedlySettings
class RefreshButtonHelper(ButtonHelper):
"""
This helper constructs a refresh button
"""
button_classnames = ['icon', 'icon-download']
def refresh_button(self, classnames_add=None, classnames_exclude=None):
if classnames_add is None: classnames_add = []
if classnames_exclude is None: classnames_exclude = []
classnames = self.button_classnames + classnames_add
cn = self.finalise_classname(classnames, classnames_exclude)
text = _('Sync {}'.format(self.verbose_name_plural.title()))
return {
'url': self.url_helper.get_action_url('refresh', query_params=self.request.GET),
'label': text, 'classname': cn, 'title': text,
}
class RefreshAdminURLHelper(AdminURLHelper):
"""
This helper constructs the different urls, to overwrite the default behaviour
and append the filters to the action.
"""
non_object_specific_actions = ('create', 'choose_parent', 'index', 'refresh')
def get_action_url(self, action, *args, **kwargs):
query_params = kwargs.pop('query_params', None)
url_name = self.get_action_url_name(action)
if action in self.non_object_specific_actions:
url = reverse(url_name)
else:
url = reverse(url_name, args=args, kwargs=kwargs)
if query_params:
url += '?{params}'.format(params=query_params.urlencode())
return url
def get_action_url_pattern(self, action):
if action in self.non_object_specific_actions:
return self._get_action_url_pattern(action)
return self._get_object_specific_action_url_pattern(action)
class RefreshView(IndexView):
"""
A Class Based View which will handle the button click
"""
# def export_csv(self):
# data = self.queryset.all()
# response = ...
# return response
@method_decorator(login_required)
def dispatch(self, request, *args, **kwargs):
super().dispatch(request, *args, **kwargs)
if not refresh_streams(FeedlySettings.for_site(request.site)):
messages.error(
request, _('Sorry, could not refresh streams. Please contact your administrator.'))
return redirect('/admin/feedler/entry/')
class EntryModelAdminMixin(object):
"""
A mixin to add to your model admin which hooks the different helpers, the view
and register the new urls.
"""
button_helper_class = RefreshButtonHelper
url_helper_class = RefreshAdminURLHelper
view_class = RefreshView
def get_admin_urls_for_registration(self):
urls = super().get_admin_urls_for_registration()
urls += (
url(
self.url_helper.get_action_url_pattern('refresh'),
self.refresh_view,
name=self.url_helper.get_action_url_name('refresh')
),
)
return urls
def refresh_view(self, request):
kwargs = {'model_admin': self}
view_class = self.view_class
return view_class.as_view(**kwargs)(request)

43
feedler/migrations/0007_auto_20171016_1059.py Normal file
View File

@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2017-10-16 08:59
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feedler', '0006_auto_20171012_1458'),
]
operations = [
migrations.RenameField(
model_name='feedlysettings',
old_name='feedly_stream',
new_name='streams',
),
migrations.RemoveField(
model_name='feedlysettings',
name='feedly_auth',
),
migrations.RemoveField(
model_name='feedlysettings',
name='feedly_pages',
),
migrations.AddField(
model_name='feedlysettings',
name='refresh',
field=models.CharField(blank=True, help_text='Refresh Token for automatic update (pro account)', max_length=1024),
),
migrations.AddField(
model_name='feedlysettings',
name='token',
field=models.CharField(blank=True, help_text='Access Token from feedly.com/v3/auth/dev', max_length=1024),
),
migrations.AlterField(
model_name='stream',
name='ident',
field=models.CharField(help_text='Example: enterprise/myuser/tag/abcd...', max_length=255),
),
]

59
feedler/models/admin.py
View File

@ -1,7 +1,5 @@
# -*- coding: utf-8 -*-
import requests, json, codecs
from django.contrib import admin
from django.db import models
@ -11,58 +9,21 @@ from django.core.mail import send_mail
from wagtail.contrib.settings.models import BaseSetting, register_setting
from .models import Entry, Stream
import feedler.feedparser as feedparser
from .models import Stream
import logging
logger = logging.getLogger('feedler')
# Feedly integration module
from feedler.refresh import refresh_streams
@register_setting
class FeedlySettings(BaseSetting):
feedly_auth = models.TextField(
help_text='Your developer authorization key', blank=True)
feedly_pages = models.IntegerField(
choices=(
(1, '2'),
(2, '5'),
(3, '10'),
(4, '50'),
), blank=True, null=True, editable=False,
help_text='How many pages to fetch?'
)
feedly_stream = models.ManyToManyField(Stream,
streams = models.ManyToManyField(Stream,
help_text='Which streams to update')
token = models.CharField(max_length=1024, blank=True,
help_text='Access Token from feedly.com/v3/auth/dev')
refresh = models.CharField(max_length=1024, blank=True,
help_text='Refresh Token for automatic update (pro account)')
class Meta:
verbose_name = 'Feedly'
API_BASEURL = 'https://cloud.feedly.com/v3/streams/contents?streamId='
@receiver(pre_save, sender=FeedlySettings)
def handle_save_settings(sender, instance, *args, **kwargs):
if instance.feedly_auth:
streams = instance.feedly_stream.all()
for stream in streams:
# Start a request to download the feed
logger.info("Processing stream %s" % stream.title)
url = API_BASEURL + stream.ident
headers = {
'Authorization': 'OAuth ' + instance.feedly_auth
}
contents = requests.get(url, headers=headers).json()
if 'errorMessage' in contents:
raise PermissionError(contents['errorMessage'])
for raw_entry in contents['items']:
eid = raw_entry['id']
# Create or update data
try:
entry = Entry.objects.get(entry_id=eid)
logger.info("Updating entry '%s'" % eid)
except Entry.DoesNotExist:
logger.info("Adding entry '%s'" % eid)
entry = Entry()
# Parse the Feedly object
entry = feedparser.parse(entry, raw_entry, stream)
# Persist resulting object
entry.save()
# @receiver(pre_save, sender=FeedlySettings)
# def handle_save_settings(sender, instance, *args, **kwargs):
# if instance.token: refresh_streams(instance)

3
feedler/models/models.py
View File

@ -10,7 +10,8 @@ from wagtail.wagtailcore.fields import RichTextField
class Stream(models.Model):
title = models.CharField(max_length=255)
ident = models.CharField(max_length=255)
ident = models.CharField(max_length=255,
help_text='Example: enterprise/myuser/tag/abcd...')
def __str__(self):
return self.title

75
feedler/refresh.py Normal file
View File

@ -0,0 +1,75 @@
# -*- coding: utf-8 -*-
import requests, json, codecs
import logging
logger = logging.getLogger('feedler')
from .models import Entry
from feedler import feedparser
API_BASEURL = 'https://cloud.feedly.com/v3/'
API_STREAMS = API_BASEURL + 'streams/contents?streamId='
API_TOKENS = API_BASEURL + 'auth/token'
def refresh_streams(settings):
# Iterate through all saved streams
logger.debug("Refreshing all streams")
for stream in settings.streams.all():
if not refresh_stream(stream, settings):
return False
return True
def refresh_token(settings):
# Request a new token
url = API_TOKENS
if not settings.refresh:
logger.warn("No Refresh token available")
return False
logger.info("Refreshing Feedly access token")
payload = {
'refresh_token': settings.refresh,
'client_id': 'feedlydev',
'client_secret': 'feedlydev',
'grant_type': 'refresh_token'
}
contents = requests.post(url, data=payload).json()
if not 'access_token' in contents or not contents['access_token']:
logger.warn("Access token could not be refreshed.")
logger.debug(contents)
return False
settings.token = contents['access_token']
settings.save()
return True
def refresh_stream(stream, settings, retry=False):
# Start a request to download the feed for a particular stream
logger.info("Processing stream %s" % stream.title)
url = API_STREAMS + stream.ident
headers = { 'Authorization': 'OAuth ' + settings.token }
contents = requests.get(url, headers=headers).json()
if 'errorMessage' in contents:
# Usually this is a token expired
if 'token expired' in contents['errorMessage'] or 'unauthorized' in contents['errorMessage']:
logger.debug(contents['errorMessage'])
if not refresh_token(settings): return False
# Make another attempt
if retry or not refresh_stream(stream, settings, True):
return False
else:
logger.error(contents['errorMessage'])
return False
for raw_entry in contents['items']:
eid = raw_entry['id']
# Create or update data
try:
entry = Entry.objects.get(entry_id=eid)
logger.info("Skipping entry '%s'" % eid)
except Entry.DoesNotExist:
logger.info("Adding entry '%s'" % eid)
entry = Entry()
# Parse the Feedly object
entry = feedparser.parse(entry, raw_entry, stream)
# Persist resulting object
entry.save()
return True

12
feedler/templates/modeladmin/feedler/entry/index.html Normal file
View File

@ -0,0 +1,12 @@
{% extends "modeladmin/index.html" %}
{% block header_extra %}
{% if user_can_create %}
<div class="right">
<div style="position: relative; top: -1px;">
{% include 'modeladmin/includes/button.html' with button=view.button_helper.refresh_button %}
</div>
</div>
{% endif %}
{{ block.super }}{% comment %}Show original buttons{% endcomment %}
{% endblock %}

5
feedler/wagtail_hooks.py
View File

@ -3,9 +3,10 @@
from wagtail.contrib.modeladmin.options import (
ModelAdmin, modeladmin_register)
from .admin import EntryModelAdminMixin
from .models import Entry, Stream
class EntryModelAdmin(ModelAdmin):
class EntryModelAdmin(EntryModelAdminMixin, ModelAdmin):
model = Entry
menu_icon = 'date'
menu_order = 200
@ -20,7 +21,7 @@ modeladmin_register(EntryModelAdmin)
class StreamModelAdmin(ModelAdmin):
model = Stream
menu_icon = 'date'
menu_order = 1000
menu_order = 900
add_to_settings_menu = True
exclude_from_explorer = True
list_display = ('title', 'ident')

34
publichealth/home/migrations/0021_auto_20171013_2321.py Normal file
View File

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2017-10-13 21:21
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('home', '0020_auto_20170920_1204'),
]
operations = [
migrations.AlterModelOptions(
name='dataletssettings',
options={'verbose_name': 'Datalets'},
),
migrations.AlterField(
model_name='dataletssettings',
name='feedback_comment',
field=models.TextField(blank=True, help_text='Any questions or general feedback', verbose_name='Comments..'),
),
migrations.AlterField(
model_name='dataletssettings',
name='feedback_question',
field=models.TextField(blank=True, help_text='Who should we reply to for questions?', verbose_name='Name'),
),
migrations.AlterField(
model_name='dataletssettings',
name='feedback_status',
field=models.IntegerField(blank=True, choices=[(1, '★'), (2, '★★'), (3, '★★★'), (4, '★★★★'), (5, '★★★★★')], help_text='How are you enjoying Wagtail?', null=True, verbose_name='Rating'),
),
]

24
publichealth/home/models/admin.py
View File

@ -9,23 +9,24 @@ from wagtail.contrib.settings.models import BaseSetting, register_setting
# A simple feedback module built into the site admin
@register_setting
@register_setting(icon='help')
class DataletsSettings(BaseSetting):
feedback_question = models.TextField(
help_text='Send us a question', blank=True)
feedback_status = models.IntegerField(
feedback_question = models.TextField(verbose_name='Name',
help_text='Who should we reply to for questions?', blank=True)
feedback_status = models.IntegerField(verbose_name='Rating',
choices=(
(1, ':-('),
(2, ':-|'),
(3, ':-)'),
(4, ':-D'),
(1, u'★'),
(2, u'★'*2),
(3, u'★'*3),
(4, u'★'*4),
(5, u'★'*5),
), blank=True, null=True,
help_text='How are you enjoying Wagtail?'
)
feedback_comment = models.TextField(
help_text='Any general feedback', blank=True)
feedback_comment = models.TextField(verbose_name='Comments..',
help_text='Any questions or general feedback', blank=True)
class Meta:
verbose_name = 'Get support'
verbose_name = 'Datalets'
@receiver(pre_save, sender=DataletsSettings)
def handle_save_settings(sender, instance, *args, **kwargs):
@ -41,3 +42,4 @@ def handle_save_settings(sender, instance, *args, **kwargs):
instance.feedback_status = None
instance.feedback_question = ""
instance.feedback_comment = ""
instance.save()

4
publichealth/settings/dev.py
View File

@ -21,6 +21,10 @@ DATABASES = {
}
}
INSTALLED_APPS = INSTALLED_APPS + [
'wagtail.contrib.wagtailstyleguide',
]
try:
from .local import *
except ImportError:

6
publichealth/settings/production.py
View File

@ -156,6 +156,12 @@ LOGGING = {
}
},
'loggers': {
'feedler': {
'handlers': [],
'level': 'INFO',
'propagate': False,
'formatter': 'verbose',
},
'publichealth': {
'handlers': [],
'level': 'INFO',

7
requirements-test.txt
View File

@ -1,3 +1,4 @@
pytest==3.0.3
pytest-splinter==1.7.6
tox==2.3.1
pytest>=3.0.3
pytest-splinter>=1.8.5
tox>=2.9.1
django-debug-toolbar>=1.8