From 065329f4713c12da1c16e89364e4742ba988e6f7 Mon Sep 17 00:00:00 2001 From: Iacopo Spalletti Date: Sun, 11 Sep 2016 20:58:15 +0200 Subject: [PATCH 1/2] Optimize querysets --- HISTORY.rst | 6 ++++++ djangocms_blog/cms_menus.py | 23 +++++++++++++++++------ djangocms_blog/models.py | 17 ++++++++++++++--- djangocms_blog/views.py | 22 ++++++++++++++++------ tests/base.py | 6 ++++++ tests/test_menu.py | 21 ++++++++++----------- tests/test_models.py | 5 +++-- 7 files changed, 72 insertions(+), 28 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1665393..5b12d2d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,12 @@ History ======= +****************** +0.8.9 (unreleased) +****************** + +* Optimized querysets + ****************** 0.8.8 (2016-09-04) ****************** diff --git a/djangocms_blog/cms_menus.py b/djangocms_blog/cms_menus.py index b105dfd..bfdf74d 100644 --- a/djangocms_blog/cms_menus.py +++ b/djangocms_blog/cms_menus.py @@ -26,6 +26,7 @@ class BlogCategoryMenu(CMSAttachMenu): Handles all types of blog menu """ name = _('Blog menu') + _config = {} def get_nodes(self, request): """ @@ -46,7 +47,11 @@ class BlogCategoryMenu(CMSAttachMenu): posts_menu = False config = False if hasattr(self, 'instance') and self.instance: - config = BlogConfig.objects.get(namespace=self.instance.application_namespace) + if not self._config.get(self.instance.application_namespace, False): + self._config[self.instance.application_namespace] = BlogConfig.objects.get( + namespace=self.instance.application_namespace + ) + config = self._config[self.instance.application_namespace] if config and config.menu_structure in (MENU_TYPE_COMPLETE, MENU_TYPE_CATEGORIES): categories_menu = True if config and config.menu_structure in (MENU_TYPE_COMPLETE, MENU_TYPE_POSTS): @@ -57,7 +62,8 @@ class BlogCategoryMenu(CMSAttachMenu): if config: categories = categories.namespace(self.instance.application_namespace) categories = categories.active_translations(language).distinct() - categories = categories.order_by('parent__id', 'translations__name') + categories = categories.order_by('parent__id', 'translations__name').\ + select_related('app_config').prefetch_related('translations') for category in categories: node = NavigationNode( category.name, @@ -65,8 +71,8 @@ class BlogCategoryMenu(CMSAttachMenu): '{0}-{1}'.format(category.__class__.__name__, category.pk), ( '{0}-{1}'.format( - category.__class__.__name__, category.parent.id - ) if category.parent else None + category.__class__.__name__, category.parent_id + ) if category.parent_id else None ) ) nodes.append(node) @@ -75,7 +81,8 @@ class BlogCategoryMenu(CMSAttachMenu): posts = Post.objects if hasattr(self, 'instance') and self.instance: posts = posts.namespace(self.instance.application_namespace) - posts = posts.active_translations(language).distinct() + posts = posts.active_translations(language).distinct().\ + select_related('app_config').prefetch_related('translations', 'categories') for post in posts: post_id = None parent = None @@ -106,6 +113,8 @@ class BlogNavModifier(Modifier): a particular blog post is viewed, a corresponding category is selected in menu """ + _config = {} + def modify(self, request, nodes, namespace, root_id, post_cut, breadcrumb): """ Actual modifier function @@ -124,7 +133,9 @@ class BlogNavModifier(Modifier): if app and app.app_config: namespace = resolve(request.path).namespace - config = app.get_config(namespace) + if not self._config.get(namespace, False): + self._config[namespace] = app.get_config(namespace) + config = self._config[namespace] try: if config and ( not isinstance(config, BlogConfig) or diff --git a/djangocms_blog/models.py b/djangocms_blog/models.py index 3a62aee..8ace18e 100644 --- a/djangocms_blog/models.py +++ b/djangocms_blog/models.py @@ -15,6 +15,7 @@ from django.db.models.signals import post_save, pre_delete from django.dispatch import receiver from django.utils import timezone from django.utils.encoding import force_bytes, force_text, python_2_unicode_compatible +from django.utils.functional import cached_property from django.utils.html import escape, strip_tags from django.utils.text import slugify from django.utils.translation import get_language, ugettext_lazy as _ @@ -76,7 +77,7 @@ class BlogCategory(TranslatableModel): verbose_name = _('blog category') verbose_name_plural = _('blog categories') - @property + @cached_property def count(self): return self.blog_posts.namespace(self.app_config.namespace).published().count() @@ -359,6 +360,16 @@ class BasePostPlugin(CMSPlugin): class Meta: abstract = True + def optimize(self, qs): + """ + Apply select_related / prefetch_related to optimize the view queries + :param qs: queryset to optimize + :return: optimized queryset + """ + return qs.select_related('app_config').prefetch_related( + 'translations', 'categories', 'categories__translations', 'categories__app_config' + ) + def post_queryset(self, request=None): language = get_language() posts = Post._default_manager @@ -367,7 +378,7 @@ class BasePostPlugin(CMSPlugin): posts = posts.active_translations(language_code=language) if not request or not getattr(request, 'toolbar', False) or not request.toolbar.edit_mode: posts = posts.published() - return posts.all() + return self.optimize(posts.all()) @python_2_unicode_compatible @@ -398,7 +409,7 @@ class LatestPostsPlugin(BasePostPlugin): posts = posts.filter(tags__in=list(self.tags.all())) if self.categories.exists(): posts = posts.filter(categories__in=list(self.categories.all())) - return posts.distinct()[:self.latest_posts] + return self.optimize(posts.distinct())[:self.latest_posts] @python_2_unicode_compatible diff --git a/djangocms_blog/views.py b/djangocms_blog/views.py index 6e09503..a18a4d6 100644 --- a/djangocms_blog/views.py +++ b/djangocms_blog/views.py @@ -21,6 +21,16 @@ User = get_user_model() class BaseBlogView(AppConfigMixin, ViewUrlMixin): model = Post + def optimize(self, qs): + """ + Apply select_related / prefetch_related to optimize the view queries + :param qs: queryset to optimize + :return: optimized queryset + """ + return qs.select_related('app_config').prefetch_related( + 'translations', 'categories', 'categories__translations', 'categories__app_config' + ) + def get_view_url(self): if not self.view_url_name: raise ImproperlyConfigured( @@ -45,7 +55,7 @@ class BaseBlogView(AppConfigMixin, ViewUrlMixin): if not getattr(self.request, 'toolbar', False) or not self.request.toolbar.edit_mode: queryset = queryset.published() setattr(self.request, get_setting('CURRENT_NAMESPACE'), self.config) - return queryset.on_site() + return self.optimize(queryset.on_site()) def get_template_names(self): template_path = (self.config and self.config.template_prefix) or 'djangocms_blog' @@ -83,7 +93,7 @@ class PostDetailView(TranslatableSlugMixin, BaseBlogView, DetailView): queryset = self.model._default_manager.all() if not getattr(self.request, 'toolbar', False) or not self.request.toolbar.edit_mode: queryset = queryset.published() - return queryset + return self.optimize(queryset) def get(self, *args, **kwargs): # submit object to cms to get corrent language switcher and selected category behavior @@ -116,7 +126,7 @@ class PostArchiveView(BaseBlogListView, ListView): qs = qs.filter(**{'%s__month' % self.date_field: self.kwargs['month']}) if 'year' in self.kwargs: qs = qs.filter(**{'%s__year' % self.date_field: self.kwargs['year']}) - return qs + return self.optimize(qs) def get_context_data(self, **kwargs): kwargs['month'] = int(self.kwargs.get('month')) if 'month' in self.kwargs else None @@ -132,7 +142,7 @@ class TaggedListView(BaseBlogListView, ListView): def get_queryset(self): qs = super(TaggedListView, self).get_queryset() - return qs.filter(tags__slug=self.kwargs['tag']) + return self.optimize(qs.filter(tags__slug=self.kwargs['tag'])) def get_context_data(self, **kwargs): kwargs['tagged_entries'] = (self.kwargs.get('tag') @@ -148,7 +158,7 @@ class AuthorEntriesView(BaseBlogListView, ListView): qs = super(AuthorEntriesView, self).get_queryset() if 'username' in self.kwargs: qs = qs.filter(**{'author__%s' % User.USERNAME_FIELD: self.kwargs['username']}) - return qs + return self.optimize(qs) def get_context_data(self, **kwargs): kwargs['author'] = User.objects.get(**{User.USERNAME_FIELD: self.kwargs.get('username')}) @@ -178,7 +188,7 @@ class CategoryEntriesView(BaseBlogListView, ListView): qs = super(CategoryEntriesView, self).get_queryset() if 'category' in self.kwargs: qs = qs.filter(categories=self.category.pk) - return qs + return self.optimize(qs) def get_context_data(self, **kwargs): kwargs['category'] = self.category diff --git a/tests/base.py b/tests/base.py index d88ea7b..674712c 100644 --- a/tests/base.py +++ b/tests/base.py @@ -12,6 +12,7 @@ from haystack.constants import DEFAULT_ALIAS from parler.utils.context import smart_override from djangocms_blog.cms_appconfig import BlogConfig +from djangocms_blog.cms_menus import BlogCategoryMenu, BlogNavModifier from djangocms_blog.models import BlogCategory, Post, ThumbnailOption User = get_user_model() @@ -214,3 +215,8 @@ class BaseTest(BaseTestCase): unified_index = search_conn.get_unified_index() index = unified_index.get_index(Post) return index + + def _reset_menus(self): + cache.clear() + BlogCategoryMenu._config = {} + BlogNavModifier._config = {} diff --git a/tests/test_menu.py b/tests/test_menu.py index 5400592..4c11398 100644 --- a/tests/test_menu.py +++ b/tests/test_menu.py @@ -2,8 +2,7 @@ from __future__ import absolute_import, print_function, unicode_literals from aldryn_apphooks_config.utils import get_app_instance -from django.core.cache import cache -from django.utils.translation import activate, override +from django.utils.translation import activate from menus.menu_pool import menu_pool from parler.utils.context import smart_override, switch_language @@ -21,7 +20,7 @@ class MenuTest(BaseTest): def setUp(self): super(MenuTest, self).setUp() self.cats = [self.category_1] - cache.clear() + self._reset_menus() for i, lang_data in enumerate(self._categories_data): cat = self._get_category(lang_data['en']) if 'it' in lang_data: @@ -34,7 +33,7 @@ class MenuTest(BaseTest): # All cms menu modifiers should be removed from menu_pool.modifiers # so that they do not interfere with our menu nodes menu_pool.modifiers = [m for m in menu_pool.modifiers if m.__module__.startswith('djangocms_blog')] - cache.clear() + self._reset_menus() def test_menu_nodes(self): """ @@ -52,7 +51,7 @@ class MenuTest(BaseTest): cats_url = set([cat.get_absolute_url() for cat in self.cats if cat.has_translation(lang)]) self.assertTrue(cats_url.issubset(nodes_url)) - cache.clear() + self._reset_menus() posts[0].categories.clear() for lang in ('en', 'it'): with smart_override(lang): @@ -82,7 +81,7 @@ class MenuTest(BaseTest): # No item in the menu self.app_config_1.app_data.config.menu_structure = MENU_TYPE_NONE self.app_config_1.save() - cache.clear() + self._reset_menus() for lang in languages: request = self.get_page_request(None, self.user, r'/%s/page-two/' % lang) with smart_override(lang): @@ -94,7 +93,7 @@ class MenuTest(BaseTest): # Only posts in the menu self.app_config_1.app_data.config.menu_structure = MENU_TYPE_POSTS self.app_config_1.save() - cache.clear() + self._reset_menus() for lang in languages: request = self.get_page_request(None, self.user, r'/%s/page-two/' % lang) with smart_override(lang): @@ -106,7 +105,7 @@ class MenuTest(BaseTest): # Only categories in the menu self.app_config_1.app_data.config.menu_structure = MENU_TYPE_CATEGORIES self.app_config_1.save() - cache.clear() + self._reset_menus() for lang in languages: request = self.get_page_request(None, self.user, r'/%s/page-two/' % lang) with smart_override(lang): @@ -118,7 +117,7 @@ class MenuTest(BaseTest): # Both types in the menu self.app_config_1.app_data.config.menu_structure = MENU_TYPE_COMPLETE self.app_config_1.save() - cache.clear() + self._reset_menus() for lang in languages: request = self.get_page_request(None, self.user, r'/%s/page-two/' % lang) with smart_override(lang): @@ -148,7 +147,7 @@ class MenuTest(BaseTest): request = self.get_page_request( pages[1], self.user, path=obj.get_absolute_url() ) - cache.clear() + self._reset_menus() menu_pool.clear(all=True) view_obj = view_cls() view_obj.request = request @@ -173,7 +172,7 @@ class MenuTest(BaseTest): request = self.get_page_request( pages[1], self.user, path=obj.get_absolute_url() ) - cache.clear() + self._reset_menus() menu_pool.clear(all=True) view_obj = view_cls() view_obj.request = request diff --git a/tests/test_models.py b/tests/test_models.py index f94c3e0..26d3c1a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -409,7 +409,7 @@ class AdminTest(BaseTest): self.assertEquals(post.sites.count(), 1) self.user.sites.clear() post.sites.clear() - post = self.reload_model(post) + self.reload_model(post) def test_admin_clear_menu(self): """ @@ -421,6 +421,7 @@ class AdminTest(BaseTest): request = self.get_page_request(None, self.user, r'/en/page-two/') first_nodes = menu_pool.get_nodes(request) + self._reset_menus() with pause_knocks(post): with self.login_user_context(self.user): data = dict(namespace='sample_app', app_title='app1', object_name='Blog') @@ -431,7 +432,7 @@ class AdminTest(BaseTest): msg_mid = MessageMiddleware() msg_mid.process_request(request) config_admin = admin.site._registry[BlogConfig] - response = config_admin.change_view(request, str(self.app_config_1.pk)) + config_admin.change_view(request, str(self.app_config_1.pk)) second_nodes = menu_pool.get_nodes(request) self.assertNotEqual(len(first_nodes), len(second_nodes)) From 53f98afd6c87c5e0944ffea718ded508826209bd Mon Sep 17 00:00:00 2001 From: Iacopo Spalletti Date: Mon, 12 Sep 2016 23:20:06 +0200 Subject: [PATCH 2/2] Changes for develop --- djangocms_blog/cms_app.py | 4 ---- djangocms_blog/cms_toolbar.py | 4 ---- tests/test_menu.py | 4 ++-- tests/test_plugins.py | 4 ++-- 4 files changed, 4 insertions(+), 12 deletions(-) delete mode 100644 djangocms_blog/cms_app.py delete mode 100644 djangocms_blog/cms_toolbar.py diff --git a/djangocms_blog/cms_app.py b/djangocms_blog/cms_app.py deleted file mode 100644 index 8c0899b..0000000 --- a/djangocms_blog/cms_app.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, unicode_literals - -from .cms_apps import * # NOQA diff --git a/djangocms_blog/cms_toolbar.py b/djangocms_blog/cms_toolbar.py deleted file mode 100644 index 733681b..0000000 --- a/djangocms_blog/cms_toolbar.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, unicode_literals - -from .cms_toolbars import * # NOQA diff --git a/tests/test_menu.py b/tests/test_menu.py index 352922e..4eaf053 100644 --- a/tests/test_menu.py +++ b/tests/test_menu.py @@ -135,7 +135,7 @@ class MenuTest(BaseTest): self.app_config_1.save() self.app_config_2.app_data.config.menu_empty_categories = False self.app_config_2.save() - cache.clear() + self._reset_menus() for lang in languages: request = self.get_page_request(None, self.user, r'/%s/page-two/' % lang) with smart_override(lang): @@ -149,7 +149,7 @@ class MenuTest(BaseTest): self.app_config_1.save() self.app_config_2.app_data.config.menu_empty_categories = True self.app_config_2.save() - cache.clear() + self._reset_menus() def test_modifier(self): """ diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 7ef7202..8d91a74 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -41,10 +41,10 @@ class PluginTest(BaseTest): plugin_nocache = add_plugin( ph, 'BlogLatestEntriesPlugin', language='en', app_config=self.app_config_1 ) - with self.assertNumQueries(53): + with self.assertNumQueries(39): plugin_nocache.render_plugin(context, ph) - with self.assertNumQueries(17): + with self.assertNumQueries(15): rendered = plugin.render_plugin(context, ph) try: self.assertTrue(rendered.find('cms_plugin-djangocms_blog-post-abstract-1') > -1)