diff --git a/publichealth/feedler/migrations/0001_initial.py b/publichealth/feedler/migrations/0001_initial.py index 9fb3abd..5a43b33 100644 --- a/publichealth/feedler/migrations/0001_initial.py +++ b/publichealth/feedler/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.3 on 2017-07-03 11:24 +# Generated by Django 1.11.3 on 2017-07-03 13:46 from __future__ import unicode_literals from django.db import migrations, models @@ -19,12 +19,12 @@ class Migration(migrations.Migration): name='Entry', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('raw', models.TextField(blank=True)), + ('raw', models.TextField(blank=True, editable=False)), ('updated', models.DateTimeField(auto_now=True)), ('published', models.DateTimeField(auto_now_add=True)), - ('entry_id', models.IntegerField(blank=True)), + ('entry_id', models.CharField(blank=True, editable=False, max_length=255, unique=True)), ('title', models.CharField(max_length=255)), - ('origin_title', models.CharField(blank=True, max_length=255)), + ('author', models.CharField(blank=True, max_length=255)), ('link', models.URLField()), ('visual', models.URLField(blank=True)), ('content', models.TextField()), @@ -34,30 +34,38 @@ class Migration(migrations.Migration): 'verbose_name_plural': 'Entries', }, ), - migrations.CreateModel( - name='EntryCategory', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('label', models.CharField(max_length=255)), - ('feedly_id', models.CharField(max_length=200)), - ], - ), migrations.CreateModel( name='FeedlySettings', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('feedly_auth', models.TextField(blank=True, help_text='Your developer authorization key')), ('feedly_pages', models.IntegerField(blank=True, choices=[(1, '2'), (2, '5'), (3, '10'), (4, '50')], help_text='How many pages to fetch?', null=True)), - ('feedly_stream', models.TextField(blank=True, help_text='Stream ID to fetch')), - ('site', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.Site')), ], options={ 'verbose_name': 'Feedly', }, ), + migrations.CreateModel( + name='Stream', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('ident', models.CharField(max_length=255)), + ], + ), + migrations.AddField( + model_name='feedlysettings', + name='feedly_stream', + field=models.ManyToManyField(to='feedler.Stream'), + ), + migrations.AddField( + model_name='feedlysettings', + name='site', + field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.Site'), + ), migrations.AddField( model_name='entry', - name='categories', - field=models.ManyToManyField(blank=True, to='feedler.EntryCategory'), + name='stream', + field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='feedler.Stream', verbose_name='Original stream'), ), ] diff --git a/publichealth/feedler/models/admin.py b/publichealth/feedler/models/admin.py index ed64f50..43c0298 100644 --- a/publichealth/feedler/models/admin.py +++ b/publichealth/feedler/models/admin.py @@ -11,6 +11,11 @@ from django.core.mail import send_mail from wagtail.contrib.settings.models import BaseSetting, register_setting +from .models import Entry, Stream + +import logging +logger = logging.getLogger('feedler') + # Feedly integration module @register_setting @@ -26,24 +31,34 @@ class FeedlySettings(BaseSetting): ), blank=True, null=True, help_text='How many pages to fetch?' ) - feedly_stream = models.TextField( - help_text='Stream ID to fetch', blank=True) + feedly_stream = models.ManyToManyField(Stream) class Meta: verbose_name = 'Feedly' +API_BASEURL = 'https://cloud.feedly.com/v3/streams/contents?streamId=' + @receiver(pre_save, sender=FeedlySettings) def handle_save_settings(sender, instance, *args, **kwargs): - if instance.feedly_stream and instance.feedly_auth: - entries = [] - url = 'https://cloud.feedly.com/v3/streams/contents?streamId=' - url = url + instance.feedly_stream - headers = { - 'Authorization': 'OAuth '+instance.feedly_auth - } - contents = requests.get(url, headers=headers).json() - if 'errorMessage' in contents: - raise PermissionError(contents['errorMessage']) - for raw_entry in contents['items']: - entry = Entry(raw_entry) - entries.append(entry) - print(json.dumps(entries)) + if instance.feedly_auth: + streams = instance.feedly_stream.all() + for stream in streams: + # Start a request to download the feed + logger.info("Processing stream %s" % stream.title) + url = API_BASEURL + stream.ident + headers = { + 'Authorization': 'OAuth ' + instance.feedly_auth + } + contents = requests.get(url, headers=headers).json() + if 'errorMessage' in contents: + raise PermissionError(contents['errorMessage']) + for raw_entry in contents['items']: + eid = raw_entry['id'] + # Create or update data + try: + entry = Entry.objects.get(entry_id=eid) + logger.info("Updating entry '%s'" % eid) + except Entry.DoesNotExist: + logger.info("Adding entry '%s'" % eid) + entry = Entry() + entry.parse(raw_entry, stream) + entry.save() diff --git a/publichealth/feedler/models/models.py b/publichealth/feedler/models/models.py index 223428a..7d2da8f 100644 --- a/publichealth/feedler/models/models.py +++ b/publichealth/feedler/models/models.py @@ -4,67 +4,79 @@ from datetime import datetime from django.db import models -class EntryCategory(models.Model): - """A structure for sorting through entry models - """ - label = models.CharField(max_length=255) - feedly_id = models.CharField(max_length=200) +class Stream(models.Model): + title = models.CharField(max_length=255) + ident = models.CharField(max_length=255) + + def __str__(self): + return self.title class Entry(models.Model): """Implementation of the Entry from the feedly API as generic Django model """ - raw = models.TextField(blank=True) - - updated = models.DateTimeField(auto_now=True) - published = models.DateTimeField(auto_now_add=True) - entry_id = models.IntegerField(blank=True) + raw = models.TextField(blank=True, editable=False) + updated = models.DateTimeField(auto_now=True, editable=False) + published = models.DateTimeField(auto_now_add=True, editable=False) + entry_id = models.CharField(max_length=255, unique=True, blank=True, editable=False) title = models.CharField(max_length=255) - origin_title = models.CharField(max_length=255, blank=True) + author = models.CharField(max_length=255, blank=True) link = models.URLField() visual = models.URLField(blank=True) content = models.TextField() tags = models.TextField(blank=True) - categories = models.ManyToManyField(EntryCategory, blank=True) + stream = models.ForeignKey(Stream, + blank=True, on_delete=models.CASCADE, + verbose_name='Original stream') class Meta: verbose_name_plural = 'Entries' - def _buildInstance(self, raw): + def parse(self, raw, stream): """ Parse the raw JSON implementation from the Feedly API """ self.raw = raw + self.stream = stream - self.published = datetime.utcfromtimestamp(raw['published']) + ts = raw['published'] / 1000 + self.published = datetime.utcfromtimestamp(ts) self.entry_id = raw['id'] self.title = raw['title'] - self.origin_title = raw['origin']['title'] - self.link = raw['alternate'][0]['href'] - self.visual = raw['visual']['url'] + + if 'author' in raw['origin']: + self.author = raw['author'] + elif 'title' in raw['origin']: + self.author = raw['origin']['title'] + + if len(raw['alternate']) > 0: + self.link = raw['alternate'][0]['href'] + if 'visual' in raw and 'url' in raw['visual']: + self.visual = raw['visual']['url'] self._buildContent() - # if 'categories' in raw: - # self.categories = raw['categories'] - # else: - # self.categories = [] - def _buildContent(self): # Collect text content if 'content' in self.raw: self.content = self.raw['content'] else: if 'summary' in self.raw: - self.content = self.raw['summary'] + if 'content' in self.raw['summary']: + self.content = self.raw['summary']['content'] + else: + self.content = self.raw['summary'] else: self.content = '' # Collect tags tags = [] for tag in self.raw['tags']: if 'label' in tag: - tags.push(tag['label'].replace(',','-')) - self.tags = ','.join('tags') + label = tag['label'].replace(',','-') + label = label.strip().lower() + if len(label) > 3 and not label in tags: + tags.append(label) + self.tags = ','.join(tags) diff --git a/publichealth/feedler/wagtail_hooks.py b/publichealth/feedler/wagtail_hooks.py index 8d1797a..dea8ddc 100644 --- a/publichealth/feedler/wagtail_hooks.py +++ b/publichealth/feedler/wagtail_hooks.py @@ -3,7 +3,7 @@ from wagtail.contrib.modeladmin.options import ( ModelAdmin, modeladmin_register) -from .models import Entry +from .models import Entry, Stream class EntryModelAdmin(ModelAdmin): model = Entry @@ -11,8 +11,18 @@ class EntryModelAdmin(ModelAdmin): menu_order = 200 add_to_settings_menu = False exclude_from_explorer = True - list_display = ('updated', 'title', 'origin_title', 'tags') - # list_filter = ('origin_title') - # search_fields = ('title', 'origin_title', 'content', 'tags') + list_display = ('updated', 'title', 'author', 'tags') + list_filter = ('author', 'tags') + search_fields = ('title', 'author', 'content', 'tags') modeladmin_register(EntryModelAdmin) + +class StreamModelAdmin(ModelAdmin): + model = Stream + menu_icon = 'date' + menu_order = 1000 + add_to_settings_menu = True + exclude_from_explorer = True + list_display = ('title', 'ident') + +modeladmin_register(StreamModelAdmin)