ungleich-public/public-health-ch
6
1
Fork 0
Code Issues Pull requests 1 Projects Releases Wiki Activity

Complete initial Feedly engine

Browse source
This commit is contained in:
Oleg Lavrovsky 2017-07-03 16:01:44 +02:00
parent dfa1606d58
commit a4e7845560
4 changed files with 106 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
publichealth/feedler/migrations/0001_initial.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-07-03 11:24 # Generated by Django 1.11.3 on 2017-07-03 13:46
from __future__ import unicode_literals from __future__ import unicode_literals
from django.db import migrations, models from django.db import migrations, models
@ -19,12 +19,12 @@ class Migration(migrations.Migration):
name='Entry', name='Entry',
fields=[ fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('raw', models.TextField(blank=True)), ('raw', models.TextField(blank=True, editable=False)),
('updated', models.DateTimeField(auto_now=True)), ('updated', models.DateTimeField(auto_now=True)),
('published', models.DateTimeField(auto_now_add=True)), ('published', models.DateTimeField(auto_now_add=True)),
('entry_id', models.IntegerField(blank=True)), ('entry_id', models.CharField(blank=True, editable=False, max_length=255, unique=True)),
('title', models.CharField(max_length=255)), ('title', models.CharField(max_length=255)),
('origin_title', models.CharField(blank=True, max_length=255)), ('author', models.CharField(blank=True, max_length=255)),
('link', models.URLField()), ('link', models.URLField()),
('visual', models.URLField(blank=True)), ('visual', models.URLField(blank=True)),
('content', models.TextField()), ('content', models.TextField()),
@ -34,30 +34,38 @@ class Migration(migrations.Migration):
'verbose_name_plural': 'Entries', 'verbose_name_plural': 'Entries',
}, },
), ),
migrations.CreateModel(
name='EntryCategory',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('label', models.CharField(max_length=255)),
('feedly_id', models.CharField(max_length=200)),
],
),
migrations.CreateModel( migrations.CreateModel(
name='FeedlySettings', name='FeedlySettings',
fields=[ fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('feedly_auth', models.TextField(blank=True, help_text='Your developer authorization key')), ('feedly_auth', models.TextField(blank=True, help_text='Your developer authorization key')),
('feedly_pages', models.IntegerField(blank=True, choices=[(1, '2'), (2, '5'), (3, '10'), (4, '50')], help_text='How many pages to fetch?', null=True)), ('feedly_pages', models.IntegerField(blank=True, choices=[(1, '2'), (2, '5'), (3, '10'), (4, '50')], help_text='How many pages to fetch?', null=True)),
('feedly_stream', models.TextField(blank=True, help_text='Stream ID to fetch')),
('site', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.Site')),
], ],
options={ options={
'verbose_name': 'Feedly', 'verbose_name': 'Feedly',
}, },
), ),
migrations.CreateModel(
name='Stream',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('title', models.CharField(max_length=255)),
('ident', models.CharField(max_length=255)),
],
),
migrations.AddField(
model_name='feedlysettings',
name='feedly_stream',
field=models.ManyToManyField(to='feedler.Stream'),
),
migrations.AddField(
model_name='feedlysettings',
name='site',
field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.Site'),
),
migrations.AddField( migrations.AddField(
model_name='entry', model_name='entry',
name='categories', name='stream',
field=models.ManyToManyField(blank=True, to='feedler.EntryCategory'), field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='feedler.Stream', verbose_name='Original stream'),
), ),
] ]

47
publichealth/feedler/models/admin.py
View file

@ -11,6 +11,11 @@ from django.core.mail import send_mail
from wagtail.contrib.settings.models import BaseSetting, register_setting from wagtail.contrib.settings.models import BaseSetting, register_setting
from .models import Entry, Stream
import logging
logger = logging.getLogger('feedler')
# Feedly integration module # Feedly integration module
@register_setting @register_setting
@ -26,24 +31,34 @@ class FeedlySettings(BaseSetting):
), blank=True, null=True, ), blank=True, null=True,
help_text='How many pages to fetch?' help_text='How many pages to fetch?'
) )
feedly_stream = models.TextField( feedly_stream = models.ManyToManyField(Stream)
help_text='Stream ID to fetch', blank=True)
class Meta: class Meta:
verbose_name = 'Feedly' verbose_name = 'Feedly'
API_BASEURL = 'https://cloud.feedly.com/v3/streams/contents?streamId='
@receiver(pre_save, sender=FeedlySettings) @receiver(pre_save, sender=FeedlySettings)
def handle_save_settings(sender, instance, *args, **kwargs): def handle_save_settings(sender, instance, *args, **kwargs):
if instance.feedly_stream and instance.feedly_auth: if instance.feedly_auth:
entries = [] streams = instance.feedly_stream.all()
url = 'https://cloud.feedly.com/v3/streams/contents?streamId=' for stream in streams:
url = url + instance.feedly_stream # Start a request to download the feed
headers = { logger.info("Processing stream %s" % stream.title)
'Authorization': 'OAuth '+instance.feedly_auth url = API_BASEURL + stream.ident
} headers = {
contents = requests.get(url, headers=headers).json() 'Authorization': 'OAuth ' + instance.feedly_auth
if 'errorMessage' in contents: }
raise PermissionError(contents['errorMessage']) contents = requests.get(url, headers=headers).json()
for raw_entry in contents['items']: if 'errorMessage' in contents:
entry = Entry(raw_entry) raise PermissionError(contents['errorMessage'])
entries.append(entry) for raw_entry in contents['items']:
print(json.dumps(entries)) eid = raw_entry['id']
# Create or update data
try:
entry = Entry.objects.get(entry_id=eid)
logger.info("Updating entry '%s'" % eid)
except Entry.DoesNotExist:
logger.info("Adding entry '%s'" % eid)
entry = Entry()
entry.parse(raw_entry, stream)
entry.save()

62
publichealth/feedler/models/models.py
View file

@ -4,67 +4,79 @@ from datetime import datetime
from django.db import models from django.db import models
class EntryCategory(models.Model): class Stream(models.Model):
"""A structure for sorting through entry models title = models.CharField(max_length=255)
""" ident = models.CharField(max_length=255)
label = models.CharField(max_length=255)
feedly_id = models.CharField(max_length=200) def __str__(self):
return self.title
class Entry(models.Model): class Entry(models.Model):
"""Implementation of the Entry from the feedly API as generic Django model """Implementation of the Entry from the feedly API as generic Django model
""" """
raw = models.TextField(blank=True) raw = models.TextField(blank=True, editable=False)
updated = models.DateTimeField(auto_now=True, editable=False)
updated = models.DateTimeField(auto_now=True) published = models.DateTimeField(auto_now_add=True, editable=False)
published = models.DateTimeField(auto_now_add=True) entry_id = models.CharField(max_length=255, unique=True, blank=True, editable=False)
entry_id = models.IntegerField(blank=True)
title = models.CharField(max_length=255) title = models.CharField(max_length=255)
origin_title = models.CharField(max_length=255, blank=True) author = models.CharField(max_length=255, blank=True)
link = models.URLField() link = models.URLField()
visual = models.URLField(blank=True) visual = models.URLField(blank=True)
content = models.TextField() content = models.TextField()
tags = models.TextField(blank=True) tags = models.TextField(blank=True)
categories = models.ManyToManyField(EntryCategory, blank=True) stream = models.ForeignKey(Stream,
blank=True, on_delete=models.CASCADE,
verbose_name='Original stream')
class Meta: class Meta:
verbose_name_plural = 'Entries' verbose_name_plural = 'Entries'
def _buildInstance(self, raw): def parse(self, raw, stream):
""" """
Parse the raw JSON implementation from the Feedly API Parse the raw JSON implementation from the Feedly API
""" """
self.raw = raw self.raw = raw
self.stream = stream
self.published = datetime.utcfromtimestamp(raw['published']) ts = raw['published'] / 1000
self.published = datetime.utcfromtimestamp(ts)
self.entry_id = raw['id'] self.entry_id = raw['id']
self.title = raw['title'] self.title = raw['title']
self.origin_title = raw['origin']['title']
self.link = raw['alternate'][0]['href'] if 'author' in raw['origin']:
self.visual = raw['visual']['url'] self.author = raw['author']
elif 'title' in raw['origin']:
self.author = raw['origin']['title']
if len(raw['alternate']) > 0:
self.link = raw['alternate'][0]['href']
if 'visual' in raw and 'url' in raw['visual']:
self.visual = raw['visual']['url']
self._buildContent() self._buildContent()
# if 'categories' in raw:
# self.categories = raw['categories']
# else:
# self.categories = []
def _buildContent(self): def _buildContent(self):
# Collect text content # Collect text content
if 'content' in self.raw: if 'content' in self.raw:
self.content = self.raw['content'] self.content = self.raw['content']
else: else:
if 'summary' in self.raw: if 'summary' in self.raw:
self.content = self.raw['summary'] if 'content' in self.raw['summary']:
self.content = self.raw['summary']['content']
else:
self.content = self.raw['summary']
else: else:
self.content = '' self.content = ''
# Collect tags # Collect tags
tags = [] tags = []
for tag in self.raw['tags']: for tag in self.raw['tags']:
if 'label' in tag: if 'label' in tag:
tags.push(tag['label'].replace(',','-')) label = tag['label'].replace(',','-')
self.tags = ','.join('tags') label = label.strip().lower()
if len(label) > 3 and not label in tags:
tags.append(label)
self.tags = ','.join(tags)

18
publichealth/feedler/wagtail_hooks.py
View file

@ -3,7 +3,7 @@
from wagtail.contrib.modeladmin.options import ( from wagtail.contrib.modeladmin.options import (
ModelAdmin, modeladmin_register) ModelAdmin, modeladmin_register)
from .models import Entry from .models import Entry, Stream
class EntryModelAdmin(ModelAdmin): class EntryModelAdmin(ModelAdmin):
model = Entry model = Entry
@ -11,8 +11,18 @@ class EntryModelAdmin(ModelAdmin):
menu_order = 200 menu_order = 200
add_to_settings_menu = False add_to_settings_menu = False
exclude_from_explorer = True exclude_from_explorer = True
list_display = ('updated', 'title', 'origin_title', 'tags') list_display = ('updated', 'title', 'author', 'tags')
# list_filter = ('origin_title') list_filter = ('author', 'tags')
# search_fields = ('title', 'origin_title', 'content', 'tags') search_fields = ('title', 'author', 'content', 'tags')
modeladmin_register(EntryModelAdmin) modeladmin_register(EntryModelAdmin)
class StreamModelAdmin(ModelAdmin):
model = Stream
menu_icon = 'date'
menu_order = 1000
add_to_settings_menu = True
exclude_from_explorer = True
list_display = ('title', 'ident')
modeladmin_register(StreamModelAdmin)