"""
This command finds and creates a report for all the usage of css rules in
an app. It aims to optimize existing codebase as well as assist the frontend
developer when designing new components by avoiding unnecessary duplication and
suggesting more/optimal alternatives.
Features:
Currently the command can find out and display:
- Media Breakpoints used in a stylesheet
- Duplicate selectors in a stylesheet
- Unused selectors
Work in progress to enable these features:
- Duplicate style declaration for same selector
- DOM validation
- Finding out dead styles (those that are always cancelled)
- Optimize media declarations
Example:
$ python manage.py optimize_frontend datacenterlight
above command produces a file ../optimize_frontend.html which contains a
report with the above mentioned features
"""
# import csv
import json
import logging
import os
import re
from collections import Counter, OrderedDict
# from itertools import zip_longest
from django import template
from django.conf import settings
from django.contrib.staticfiles import finders
from django.core.management.base import BaseCommand
logger = logging.getLogger(__name__)
RE_PATTERNS = {
'view_html': '[\'\"](.*\.html)',
'html_html': '{% (?:extends|include) [\'\"]?(.*\.html)',
'html_style': '{% static [\'\"]?(.*\.css)',
'css_media': (
'^\s*\@media([^{]+)\{\s*([\s\S]*?})\s*}'
),
'css_selector': (
'^\s*([.#\[:_A-Za-z][^{]*?)\s*'
'\s*{\s*([\s\S]*?)\s*}'
),
'html_class': 'class=[\'\"]([a-zA-Z0-9-_\s]*)',
'html_id': 'id=[\'\"]([a-zA-Z0-9-_]*)'
}
class Command(BaseCommand):
help = (
'Finds unused and duplicate style declarations from the stylesheets '
'used in the templates of each app'
)
requires_system_checks = False
def add_arguments(self, parser):
# positional arguments
parser.add_argument(
'apps', nargs='+', type=str,
help='name of the apps to be optimized'
)
# Named (optional) arguments
parser.add_argument(
'--together',
action='store_true',
help='optimize the apps together'
)
parser.add_argument(
'--css',
action='store_true',
help='optimize only the css rules declared in each stylesheet'
)
def handle(self, *args, **options):
apps_list = options['apps']
report = {}
for app in apps_list:
if options['css']:
report[app] = self.optimize_css(app)
# write report
write_report(report)
def optimize_css(self, app_name):
"""Optimize declarations inside a css stylesheet
Args:
app_name (str): The application name
"""
# get html and css files used in the app
files = get_files(app_name)
# get_selectors_from_css
css_selectors = get_selectors_css(files['style'])
# get_selectors_from_html
html_selectors = get_selectors_html(files['html'])
report = {
'css_dup': get_css_duplication(css_selectors),
'css_unused': get_css_unused(css_selectors, html_selectors)
}
return report
def get_files(app_name):
"""Get all the `html` and `css` files used in an app.
Args:
app_name (str): The application name
Returns:
dict: A dictonary containing Counter of occurence of each
html and css file in `html` and `style` fields respectively.
For example:
{
'html': {'datacenterlight/success.html': 1},
'style': {'datacenterlight/css/bootstrap.min.css': 2}
}
"""
# the view file for the app
app_view = os.path.join(settings.PROJECT_DIR, app_name, 'views.py')
# get template files called from the view
all_html_list = file_match_pattern(app_view, 'view_html')
# list of unique template files
uniq_html_list = list(OrderedDict.fromkeys(all_html_list).keys())
# list of stylesheets
all_style_list = []
file_patterns = ['html_html', 'html_style']
# get html and css files called from within templates
i = 0
while i < len(uniq_html_list):
template_name = uniq_html_list[i]
try:
temp_files = templates_match_pattern(
template_name, file_patterns
)
except template.exceptions.TemplateDoesNotExist as e:
print("template file not found: ", str(e))
all_html_list = [
h for h in all_html_list if h != template_name
]
del uniq_html_list[i]
else:
all_html_list.extend(temp_files[0])
uniq_html_list = list(
OrderedDict.fromkeys(all_html_list).keys()
)
all_style_list.extend(temp_files[1])
i += 1
# counter dict for the html files called from view
result = {
'html': Counter(all_html_list),
'style': Counter(all_style_list)
}
# print(result)
return result
def get_selectors_css(files):
"""Gets the selectors and declarations from a stylesheet.
Args:
files (list): A list of path of stylesheets.
Returns:
dict: A nested dictionary with the structre as
`{'file': {'media-selector': [('selectors',`declarations')]}}`
For example:
{
'datacenterlight/css/landing-page.css':{
'(min-width: 768px)': [
('.lead-right', 'text-align: right;'),
]
}
}
"""
selectors = {}
media_selectors = {}
# get media selectors and other simple declarations
for file in files:
if any(vendor in file for vendor in ['bootstrap', 'font-awesome']):
continue
result = finders.find(file)
if result:
with open(result) as f:
data = f.read()
media_selectors[file] = string_match_pattern(data, 'css_media')
new_data = string_remove_pattern(data, 'css_media')
default_match = string_match_pattern(new_data, 'css_selector')
selectors[file] = {
'default': [
[' '.join(grp.split()) for grp in m] for m in default_match
]
}
# get declarations from media queries
for file, match_list in media_selectors.items():
for match in match_list:
query = match[0]
block_text = ' '.join(match[1].split())
results = string_match_pattern(
block_text, 'css_selector'
)
f_query = ' '.join(query.replace(':', ': ').split())
if f_query in selectors[file]:
selectors[file][f_query].extend(results)
else:
selectors[file][f_query] = results
return selectors
def get_selectors_html(files):
"""Get `class` and `id` used in html files.
Args:
files (list): A list of html files path.
Returns:
dict: a dictonary of all the classes and ids found in the file, in
`class` and `id` field respectively.
"""
selectors = {}
for file in files:
results = templates_match_pattern(file, ['html_class', 'html_id'])
class_dict = {c: 1 for match in results[0] for c in match.split()}
selectors[file] = {
'classes': list(class_dict.keys()),
'ids': results[1],
}
return selectors
def file_match_pattern(file, patterns):
"""Match a regex pattern in a file
Args:
file (str): Complete path of file
patterns (list or str): The pattern(s) to be searched in the file
Returns:
list: A list of all the matches in the file. Each item is a list of
all the captured groups in the pattern. If multiple patterns are given,
the returned list is a list of such lists.
For example:
[('.lead', 'font-size: 18px;'), ('.btn-lg', 'min-width: 180px;')]
"""
with open(file) as f:
data = f.read()
results = string_match_pattern(data, patterns)
return results
def string_match_pattern(data, patterns):
"""Match a regex pattern in a string
Args:
data (str): the string to search for the pattern
patterns (list or str): The pattern(s) to be searched in the file
Returns:
list: A list of all the matches in the string. Each item is a list of
all the captured groups in the pattern. If multiple patterns are given,
the returned list is a list of such lists.
For example:
[('.lead', 'font-size: 18px;'), ('.btn-lg', 'min-width: 180px;')]
"""
if not isinstance(patterns, str):
results = []
for p in patterns:
re_pattern = re.compile(RE_PATTERNS[p], re.MULTILINE)
results.append(re.findall(re_pattern, data))
else:
re_pattern = re.compile(RE_PATTERNS[patterns], re.MULTILINE)
results = re.findall(re_pattern, data)
return results
def string_remove_pattern(data, patterns):
"""Remove a pattern from a string
Args:
data (str): the string to search for the patter
patterns (list or str): The pattern(s) to be removed from the file
Returns:
str: The new string with all instance of matching pattern
removed from it
"""
if not isinstance(patterns, str):
for p in patterns:
re_pattern = re.compile(RE_PATTERNS[p], re.MULTILINE)
data = re.sub(re_pattern, '', data)
else:
re_pattern = re.compile(RE_PATTERNS[patterns], re.MULTILINE)
data = re.sub(re_pattern, '', data)
return data
def templates_match_pattern(template_name, patterns):
"""Match a regex pattern in the first found template file
Args:
file (str): Path of template file
patterns (list or str): The pattern(s) to be searched in the file
Returns:
list: A list of all the matches in the file. Each item is a list of
all the captured groups in the pattern. If multiple patterns are given,
the returned list is a list of such lists.
For example:
[('.lead', 'font-size: 18px;'), ('.btn-lg', 'min-width: 180px;')]
"""
t = template.loader.get_template(template_name)
data = t.template.source
results = string_match_pattern(data, patterns)
return results
def get_css_duplication(css_selectors):
"""Get duplicate selectors from the same stylesheet
Args:
css_selectors (dict): A dictonary containing css selectors from
all the files in the app in the below structure.
`{'file': {'media-selector': [('selectors',`declarations')]}}`
Returns:
dict: A dictonary containing the count of any duplicate selector in
each file.
`{'file': {'media-selector': {'selector': count}}}`
"""
# duplicate css selectors in stylesheets
rule_count = {}
for file, media_selectors in css_selectors.items():
rule_count[file] = {}
for media, rules in media_selectors.items():
rules_dict = Counter([rule[0] for rule in rules])
dup_rules_dict = {k: v for k, v in rules_dict.items() if v > 1}
if dup_rules_dict:
rule_count[file][media] = dup_rules_dict
return rule_count
def get_css_unused(css_selectors, html_selectors):
"""Get selectors from stylesheets that are not used in any of the html
files in which the stylesheet is used.
Args:
css_selectors (dict): A dictonary containing css selectors from
all the files in the app in the below structure.
`{'file': {'media-selector': [('selectors',`declarations')]}}`
html_selectors (dict): A dictonary containing the 'class' and 'id'
declarations from all html files
"""
with open('utils/optimize/test.json', 'w') as f:
json.dump([html_selectors, css_selectors], f, indent=4)
# print(html_selectors, css_selectors)
def write_report(all_reports, filename='frontend'):
"""Write the generated report to a file for re-use
Args;
all_reports (dict): A dictonary of report obtained from different tests
filename (str): An optional suffix for the output file
"""
# full_filename = 'utils/optimize/optimize_' + filename + '.html'
# output_file = os.path.join(
# settings.PROJECT_DIR, full_filename
# )
with open('utils/optimize/op_frontend.json', 'w') as f:
json.dump(all_reports, f, indent=4)
# with open(output_file, 'w', newline='') as f:
# f.write(
# template.loader.render_to_string(
# 'utils/report.html', {'all_reports': all_reports}
# )
# )
# w = csv.writer(f)
# print(zip_longest(*results))
# for r in zip_longest(*results):
# w.writerow(r)
# a list of all the html tags (to be moved in a json file)
html_tags = [
"a",
"abbr",
"address",
"article",
"area",
"aside",
"audio",
"b",
"base",
"bdi",
"bdo",
"blockquote",
"body",
"br",
"button",
"canvas",
"caption",
"cite",
"code",
"col",
"colgroup",
"datalist",
"dd",
"del",
"details",
"dfn",
"div",
"dl",
"dt",
"em",
"embed",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"head",
"header",
"hgroup",
"hr",
"html",
"i",
"iframe",
"img",
"input",
"ins",
"kbd",
"keygen",
"label",
"legend",
"li",
"link",
"map",
"mark",
"menu",
"meta",
"meter",
"nav",
"noscript",
"object",
"ol",
"optgroup",
"option",
"output",
"p",
"param",
"pre",
"progress",
"q",
"rp",
"rt",
"ruby",
"s",
"samp",
"script",
"section",
"select",
"source",
"small",
"span",
"strong",
"style",
"sub",
"summary",
"sup",
"textarea",
"table",
"tbody",
"td",
"tfoot",
"thead",
"th",
"time",
"title",
"tr",
"u",
"ul",
"var",
"video",
"wbr"
]