From 4f2562cbc30922a8460898ded611ff03b7e2f47d Mon Sep 17 00:00:00 2001 From: Oleg Lavrovsky Date: Fri, 13 Oct 2017 15:40:24 +0200 Subject: [PATCH] Token refresh --- feedler/migrations/0007_auto_20171013_1515.py | 33 +++++++++ feedler/models/admin.py | 60 ++------------- feedler/refresh.py | 73 +++++++++++++++++++ publichealth/settings/production.py | 6 ++ 4 files changed, 118 insertions(+), 54 deletions(-) create mode 100644 feedler/migrations/0007_auto_20171013_1515.py create mode 100644 feedler/refresh.py diff --git a/feedler/migrations/0007_auto_20171013_1515.py b/feedler/migrations/0007_auto_20171013_1515.py new file mode 100644 index 0000000..458fffd --- /dev/null +++ b/feedler/migrations/0007_auto_20171013_1515.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2017-10-13 13:15 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('feedler', '0006_auto_20171012_1458'), + ] + + operations = [ + migrations.RenameField( + model_name='feedlysettings', + old_name='feedly_stream', + new_name='streams', + ), + migrations.RemoveField( + model_name='feedlysettings', + name='feedly_auth', + ), + migrations.RemoveField( + model_name='feedlysettings', + name='feedly_pages', + ), + migrations.AddField( + model_name='feedlysettings', + name='token', + field=models.CharField(blank=True, help_text='Access Token from developer.feedly.com', max_length=255), + ), + ] diff --git a/feedler/models/admin.py b/feedler/models/admin.py index cabc088..43c5972 100644 --- a/feedler/models/admin.py +++ b/feedler/models/admin.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -import requests, json, codecs - from django.contrib import admin from django.db import models @@ -10,66 +8,20 @@ from django.dispatch import receiver from django.core.mail import send_mail from wagtail.contrib.settings.models import BaseSetting, register_setting -from wagtail.wagtailadmin import messages -from .models import Entry, Stream -import feedler.feedparser as feedparser +from .models import Stream -import logging -logger = logging.getLogger('feedler') - -# Feedly integration module +from feedler.refresh import refresh_streams @register_setting class FeedlySettings(BaseSetting): - feedly_auth = models.TextField( - help_text='Your developer authorization key', blank=True) - feedly_pages = models.IntegerField( - choices=( - (1, '2'), - (2, '5'), - (3, '10'), - (4, '50'), - ), blank=True, null=True, editable=False, - help_text='How many pages to fetch?' - ) - feedly_stream = models.ManyToManyField(Stream, + streams = models.ManyToManyField(Stream, help_text='Which streams to update') + token = models.CharField(max_length=255, blank=True, + help_text='Access Token from developer.feedly.com') class Meta: verbose_name = 'Feedly' -API_BASEURL = 'https://cloud.feedly.com/v3/streams/contents?streamId=' - @receiver(pre_save, sender=FeedlySettings) def handle_save_settings(sender, instance, *args, **kwargs): - if instance.feedly_auth: - streams = instance.feedly_stream.all() - for stream in streams: - # Start a request to download the feed - logger.info("Processing stream %s" % stream.title) - url = API_BASEURL + stream.ident - headers = { - 'Authorization': 'OAuth ' + instance.feedly_auth - } - contents = requests.get(url, headers=headers).json() - if 'errorMessage' in contents: - # Usually this is a token expired - if 'token expired' in contents['errorMessage']: - # TODO: request new token - pass - logger.error(contents['errorMessage']) - messages.error(sender, "Failed to fetch items: %s" % contents['errorMessage']) - return - for raw_entry in contents['items']: - eid = raw_entry['id'] - # Create or update data - try: - entry = Entry.objects.get(entry_id=eid) - logger.info("Updating entry '%s'" % eid) - except Entry.DoesNotExist: - logger.info("Adding entry '%s'" % eid) - entry = Entry() - # Parse the Feedly object - entry = feedparser.parse(entry, raw_entry, stream) - # Persist resulting object - entry.save() + if instance.token: refresh_streams(instance) diff --git a/feedler/refresh.py b/feedler/refresh.py new file mode 100644 index 0000000..e361358 --- /dev/null +++ b/feedler/refresh.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +import requests, json, codecs + +import logging +logger = logging.getLogger('feedler') + +from .models import Entry +from feedler import feedparser + +API_BASEURL = 'https://cloud.feedly.com/v3/' +API_STREAMS = API_BASEURL + 'streams/contents?streamId=' +API_TOKENS = API_BASEURL + 'auth/token' + +def refresh_streams(settings): + # Iterate through all saved streams + logger.warn("Refreshing all streams") + for stream in settings.streams.all(): + if not refresh_stream(stream, settings): + return False + return True + +def get_headers(settings): + return { + 'Authorization': 'OAuth ' + settings.token + } + +def refresh_token(settings): + # Request a new token + url = API_TOKENS + logger.warn("Refreshing Feedly access token") + payload = { + 'refresh_token': settings.token, + 'client_id': 'feedlydev', + 'client_secret': 'feedlydev', + 'grant_type': 'refresh_token' + } + contents = requests.get(url, data=payload, headers=get_headers(settings)).json() + if not 'access_token' in contents or not contents['access_token']: + logger.error("Access token could not be refreshed.") + return False + settings.token = contents['access_token'] + settings.save() + return True + +def refresh_stream(stream, settings, retry=False): + # Start a request to download the feed for a particular stream + logger.warn("Processing stream %s" % stream.title) + url = API_STREAMS + stream.ident + contents = requests.get(url, headers=get_headers(settings)).json() + if 'errorMessage' in contents: + # Usually this is a token expired + if 'token expired' in contents['errorMessage'] or 'unauthorized' in contents['errorMessage']: + if not refresh_token(settings): return False + # Make another attempt + if retry or not refresh_stream(stream, settings, True): + return False + else: + logger.error(contents['errorMessage']) + return False + for raw_entry in contents['items']: + eid = raw_entry['id'] + # Create or update data + try: + entry = Entry.objects.get(entry_id=eid) + logger.info("Updating entry '%s'" % eid) + except Entry.DoesNotExist: + logger.info("Adding entry '%s'" % eid) + entry = Entry() + # Parse the Feedly object + entry = feedparser.parse(entry, raw_entry, stream) + # Persist resulting object + entry.save() diff --git a/publichealth/settings/production.py b/publichealth/settings/production.py index eb23ad7..7b753de 100644 --- a/publichealth/settings/production.py +++ b/publichealth/settings/production.py @@ -156,6 +156,12 @@ LOGGING = { } }, 'loggers': { + 'feedler': { + 'handlers': [], + 'level': 'INFO', + 'propagate': False, + 'formatter': 'verbose', + }, 'publichealth': { 'handlers': [], 'level': 'INFO',