Complete initial Feedly engine
This commit is contained in:
Oleg Lavrovsky
parent
dfa1606d58
commit
a4e7845560
4 changed files with 106 additions and 61 deletions
|
|
@ -1,5 +1,5 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.11.3 on 2017-07-03 11:24
|
||||
# Generated by Django 1.11.3 on 2017-07-03 13:46
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import migrations, models
|
||||
|
|
@ -19,12 +19,12 @@ class Migration(migrations.Migration):
|
|||
name='Entry',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('raw', models.TextField(blank=True)),
|
||||
('raw', models.TextField(blank=True, editable=False)),
|
||||
('updated', models.DateTimeField(auto_now=True)),
|
||||
('published', models.DateTimeField(auto_now_add=True)),
|
||||
('entry_id', models.IntegerField(blank=True)),
|
||||
('entry_id', models.CharField(blank=True, editable=False, max_length=255, unique=True)),
|
||||
('title', models.CharField(max_length=255)),
|
||||
('origin_title', models.CharField(blank=True, max_length=255)),
|
||||
('author', models.CharField(blank=True, max_length=255)),
|
||||
('link', models.URLField()),
|
||||
('visual', models.URLField(blank=True)),
|
||||
('content', models.TextField()),
|
||||
|
|
@ -34,30 +34,38 @@ class Migration(migrations.Migration):
|
|||
'verbose_name_plural': 'Entries',
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='EntryCategory',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('label', models.CharField(max_length=255)),
|
||||
('feedly_id', models.CharField(max_length=200)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='FeedlySettings',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('feedly_auth', models.TextField(blank=True, help_text='Your developer authorization key')),
|
||||
('feedly_pages', models.IntegerField(blank=True, choices=[(1, '2'), (2, '5'), (3, '10'), (4, '50')], help_text='How many pages to fetch?', null=True)),
|
||||
('feedly_stream', models.TextField(blank=True, help_text='Stream ID to fetch')),
|
||||
('site', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.Site')),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'Feedly',
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Stream',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('title', models.CharField(max_length=255)),
|
||||
('ident', models.CharField(max_length=255)),
|
||||
],
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='feedlysettings',
|
||||
name='feedly_stream',
|
||||
field=models.ManyToManyField(to='feedler.Stream'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='feedlysettings',
|
||||
name='site',
|
||||
field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.Site'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='entry',
|
||||
name='categories',
|
||||
field=models.ManyToManyField(blank=True, to='feedler.EntryCategory'),
|
||||
name='stream',
|
||||
field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='feedler.Stream', verbose_name='Original stream'),
|
||||
),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -11,6 +11,11 @@ from django.core.mail import send_mail
|
|||
|
||||
from wagtail.contrib.settings.models import BaseSetting, register_setting
|
||||
|
||||
from .models import Entry, Stream
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger('feedler')
|
||||
|
||||
# Feedly integration module
|
||||
|
||||
@register_setting
|
||||
|
|
@ -26,24 +31,34 @@ class FeedlySettings(BaseSetting):
|
|||
), blank=True, null=True,
|
||||
help_text='How many pages to fetch?'
|
||||
)
|
||||
feedly_stream = models.TextField(
|
||||
help_text='Stream ID to fetch', blank=True)
|
||||
feedly_stream = models.ManyToManyField(Stream)
|
||||
class Meta:
|
||||
verbose_name = 'Feedly'
|
||||
|
||||
API_BASEURL = 'https://cloud.feedly.com/v3/streams/contents?streamId='
|
||||
|
||||
@receiver(pre_save, sender=FeedlySettings)
|
||||
def handle_save_settings(sender, instance, *args, **kwargs):
|
||||
if instance.feedly_stream and instance.feedly_auth:
|
||||
entries = []
|
||||
url = 'https://cloud.feedly.com/v3/streams/contents?streamId='
|
||||
url = url + instance.feedly_stream
|
||||
headers = {
|
||||
'Authorization': 'OAuth '+instance.feedly_auth
|
||||
}
|
||||
contents = requests.get(url, headers=headers).json()
|
||||
if 'errorMessage' in contents:
|
||||
raise PermissionError(contents['errorMessage'])
|
||||
for raw_entry in contents['items']:
|
||||
entry = Entry(raw_entry)
|
||||
entries.append(entry)
|
||||
print(json.dumps(entries))
|
||||
if instance.feedly_auth:
|
||||
streams = instance.feedly_stream.all()
|
||||
for stream in streams:
|
||||
# Start a request to download the feed
|
||||
logger.info("Processing stream %s" % stream.title)
|
||||
url = API_BASEURL + stream.ident
|
||||
headers = {
|
||||
'Authorization': 'OAuth ' + instance.feedly_auth
|
||||
}
|
||||
contents = requests.get(url, headers=headers).json()
|
||||
if 'errorMessage' in contents:
|
||||
raise PermissionError(contents['errorMessage'])
|
||||
for raw_entry in contents['items']:
|
||||
eid = raw_entry['id']
|
||||
# Create or update data
|
||||
try:
|
||||
entry = Entry.objects.get(entry_id=eid)
|
||||
logger.info("Updating entry '%s'" % eid)
|
||||
except Entry.DoesNotExist:
|
||||
logger.info("Adding entry '%s'" % eid)
|
||||
entry = Entry()
|
||||
entry.parse(raw_entry, stream)
|
||||
entry.save()
|
||||
|
|
|
|||
|
|
@ -4,67 +4,79 @@ from datetime import datetime
|
|||
|
||||
from django.db import models
|
||||
|
||||
class EntryCategory(models.Model):
|
||||
"""A structure for sorting through entry models
|
||||
"""
|
||||
label = models.CharField(max_length=255)
|
||||
feedly_id = models.CharField(max_length=200)
|
||||
class Stream(models.Model):
|
||||
title = models.CharField(max_length=255)
|
||||
ident = models.CharField(max_length=255)
|
||||
|
||||
def __str__(self):
|
||||
return self.title
|
||||
|
||||
class Entry(models.Model):
|
||||
"""Implementation of the Entry from the feedly API as generic Django model
|
||||
"""
|
||||
raw = models.TextField(blank=True)
|
||||
|
||||
updated = models.DateTimeField(auto_now=True)
|
||||
published = models.DateTimeField(auto_now_add=True)
|
||||
entry_id = models.IntegerField(blank=True)
|
||||
raw = models.TextField(blank=True, editable=False)
|
||||
updated = models.DateTimeField(auto_now=True, editable=False)
|
||||
published = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
entry_id = models.CharField(max_length=255, unique=True, blank=True, editable=False)
|
||||
|
||||
title = models.CharField(max_length=255)
|
||||
origin_title = models.CharField(max_length=255, blank=True)
|
||||
author = models.CharField(max_length=255, blank=True)
|
||||
link = models.URLField()
|
||||
visual = models.URLField(blank=True)
|
||||
|
||||
content = models.TextField()
|
||||
tags = models.TextField(blank=True)
|
||||
|
||||
categories = models.ManyToManyField(EntryCategory, blank=True)
|
||||
stream = models.ForeignKey(Stream,
|
||||
blank=True, on_delete=models.CASCADE,
|
||||
verbose_name='Original stream')
|
||||
|
||||
class Meta:
|
||||
verbose_name_plural = 'Entries'
|
||||
|
||||
def _buildInstance(self, raw):
|
||||
def parse(self, raw, stream):
|
||||
"""
|
||||
Parse the raw JSON implementation from the Feedly API
|
||||
"""
|
||||
self.raw = raw
|
||||
self.stream = stream
|
||||
|
||||
self.published = datetime.utcfromtimestamp(raw['published'])
|
||||
ts = raw['published'] / 1000
|
||||
self.published = datetime.utcfromtimestamp(ts)
|
||||
self.entry_id = raw['id']
|
||||
|
||||
self.title = raw['title']
|
||||
self.origin_title = raw['origin']['title']
|
||||
self.link = raw['alternate'][0]['href']
|
||||
self.visual = raw['visual']['url']
|
||||
|
||||
if 'author' in raw['origin']:
|
||||
self.author = raw['author']
|
||||
elif 'title' in raw['origin']:
|
||||
self.author = raw['origin']['title']
|
||||
|
||||
if len(raw['alternate']) > 0:
|
||||
self.link = raw['alternate'][0]['href']
|
||||
if 'visual' in raw and 'url' in raw['visual']:
|
||||
self.visual = raw['visual']['url']
|
||||
|
||||
self._buildContent()
|
||||
|
||||
# if 'categories' in raw:
|
||||
# self.categories = raw['categories']
|
||||
# else:
|
||||
# self.categories = []
|
||||
|
||||
def _buildContent(self):
|
||||
# Collect text content
|
||||
if 'content' in self.raw:
|
||||
self.content = self.raw['content']
|
||||
else:
|
||||
if 'summary' in self.raw:
|
||||
self.content = self.raw['summary']
|
||||
if 'content' in self.raw['summary']:
|
||||
self.content = self.raw['summary']['content']
|
||||
else:
|
||||
self.content = self.raw['summary']
|
||||
else:
|
||||
self.content = ''
|
||||
# Collect tags
|
||||
tags = []
|
||||
for tag in self.raw['tags']:
|
||||
if 'label' in tag:
|
||||
tags.push(tag['label'].replace(',','-'))
|
||||
self.tags = ','.join('tags')
|
||||
label = tag['label'].replace(',','-')
|
||||
label = label.strip().lower()
|
||||
if len(label) > 3 and not label in tags:
|
||||
tags.append(label)
|
||||
self.tags = ','.join(tags)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from wagtail.contrib.modeladmin.options import (
|
||||
ModelAdmin, modeladmin_register)
|
||||
|
||||
from .models import Entry
|
||||
from .models import Entry, Stream
|
||||
|
||||
class EntryModelAdmin(ModelAdmin):
|
||||
model = Entry
|
||||
|
|
@ -11,8 +11,18 @@ class EntryModelAdmin(ModelAdmin):
|
|||
menu_order = 200
|
||||
add_to_settings_menu = False
|
||||
exclude_from_explorer = True
|
||||
list_display = ('updated', 'title', 'origin_title', 'tags')
|
||||
# list_filter = ('origin_title')
|
||||
# search_fields = ('title', 'origin_title', 'content', 'tags')
|
||||
list_display = ('updated', 'title', 'author', 'tags')
|
||||
list_filter = ('author', 'tags')
|
||||
search_fields = ('title', 'author', 'content', 'tags')
|
||||
|
||||
modeladmin_register(EntryModelAdmin)
|
||||
|
||||
class StreamModelAdmin(ModelAdmin):
|
||||
model = Stream
|
||||
menu_icon = 'date'
|
||||
menu_order = 1000
|
||||
add_to_settings_menu = True
|
||||
exclude_from_explorer = True
|
||||
list_display = ('title', 'ident')
|
||||
|
||||
modeladmin_register(StreamModelAdmin)
|
||||
|
|
|
|||
Loading…
Reference in a new issue