498 lines
14 KiB
Python
498 lines
14 KiB
Python
"""
|
|
This command finds and creates a report for all the usage of css rules in
|
|
an app. It aims to optimize existing codebase as well as assist the frontend
|
|
developer when designing new components by avoiding unnecessary duplication and
|
|
suggesting more/optimal alternatives.
|
|
|
|
Features:
|
|
Currently the command can find out and display:
|
|
- Media Breakpoints used in a stylesheet
|
|
- Duplicate selectors in a stylesheet
|
|
- Unused selectors
|
|
Work in progress to enable these features:
|
|
- Duplicate style declaration for same selector
|
|
- DOM validation
|
|
- Finding out dead styles (those that are always cancelled)
|
|
- Optimize media declarations
|
|
|
|
Example:
|
|
$ python manage.py optimize_frontend datacenterlight
|
|
above command produces a file ../optimize_frontend.html which contains a
|
|
report with the above mentioned features
|
|
"""
|
|
|
|
# import csv
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from collections import Counter, OrderedDict
|
|
# from itertools import zip_longest
|
|
|
|
from django import template
|
|
from django.conf import settings
|
|
from django.contrib.staticfiles import finders
|
|
from django.core.management.base import BaseCommand
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
RE_PATTERNS = {
|
|
'view_html': '[\'\"](.*\.html)',
|
|
'html_html': '{% (?:extends|include) [\'\"]?(.*\.html)',
|
|
'html_style': '{% static [\'\"]?(.*\.css)',
|
|
'css_media': (
|
|
'^\s*\@media([^{]+)\{\s*([\s\S]*?})\s*}'
|
|
),
|
|
'css_selector': (
|
|
'^\s*([.#\[:_A-Za-z][^{]*?)\s*'
|
|
'\s*{\s*([\s\S]*?)\s*}'
|
|
),
|
|
'html_class': 'class=[\'\"]([a-zA-Z0-9-_\s]*)',
|
|
'html_id': 'id=[\'\"]([a-zA-Z0-9-_]*)'
|
|
}
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = (
|
|
'Finds unused and duplicate style declarations from the stylesheets '
|
|
'used in the templates of each app'
|
|
)
|
|
requires_system_checks = False
|
|
|
|
def add_arguments(self, parser):
|
|
# positional arguments
|
|
parser.add_argument(
|
|
'apps', nargs='+', type=str,
|
|
help='name of the apps to be optimized'
|
|
)
|
|
|
|
# Named (optional) arguments
|
|
parser.add_argument(
|
|
'--together',
|
|
action='store_true',
|
|
help='optimize the apps together'
|
|
)
|
|
parser.add_argument(
|
|
'--css',
|
|
action='store_true',
|
|
help='optimize only the css rules declared in each stylesheet'
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
apps_list = options['apps']
|
|
report = {}
|
|
for app in apps_list:
|
|
if options['css']:
|
|
report[app] = self.optimize_css(app)
|
|
# write report
|
|
write_report(report)
|
|
|
|
def optimize_css(self, app_name):
|
|
"""Optimize declarations inside a css stylesheet
|
|
|
|
Args:
|
|
app_name (str): The application name
|
|
"""
|
|
# get html and css files used in the app
|
|
files = get_files(app_name)
|
|
# get_selectors_from_css
|
|
css_selectors = get_selectors_css(files['style'])
|
|
# get_selectors_from_html
|
|
html_selectors = get_selectors_html(files['html'])
|
|
report = {
|
|
'css_dup': get_css_duplication(css_selectors),
|
|
'css_unused': get_css_unused(css_selectors, html_selectors)
|
|
}
|
|
return report
|
|
|
|
|
|
def get_files(app_name):
|
|
"""Get all the `html` and `css` files used in an app.
|
|
|
|
Args:
|
|
app_name (str): The application name
|
|
|
|
Returns:
|
|
dict: A dictonary containing Counter of occurence of each
|
|
html and css file in `html` and `style` fields respectively.
|
|
For example:
|
|
{
|
|
'html': {'datacenterlight/success.html': 1},
|
|
'style': {'datacenterlight/css/bootstrap.min.css': 2}
|
|
}
|
|
"""
|
|
# the view file for the app
|
|
app_view = os.path.join(settings.PROJECT_DIR, app_name, 'views.py')
|
|
# get template files called from the view
|
|
all_html_list = file_match_pattern(app_view, 'view_html')
|
|
# list of unique template files
|
|
uniq_html_list = list(OrderedDict.fromkeys(all_html_list).keys())
|
|
# list of stylesheets
|
|
all_style_list = []
|
|
file_patterns = ['html_html', 'html_style']
|
|
# get html and css files called from within templates
|
|
i = 0
|
|
while i < len(uniq_html_list):
|
|
template_name = uniq_html_list[i]
|
|
try:
|
|
temp_files = templates_match_pattern(
|
|
template_name, file_patterns
|
|
)
|
|
except template.exceptions.TemplateDoesNotExist as e:
|
|
print("template file not found: ", str(e))
|
|
all_html_list = [
|
|
h for h in all_html_list if h != template_name
|
|
]
|
|
del uniq_html_list[i]
|
|
else:
|
|
all_html_list.extend(temp_files[0])
|
|
uniq_html_list = list(
|
|
OrderedDict.fromkeys(all_html_list).keys()
|
|
)
|
|
all_style_list.extend(temp_files[1])
|
|
i += 1
|
|
# counter dict for the html files called from view
|
|
result = {
|
|
'html': Counter(all_html_list),
|
|
'style': Counter(all_style_list)
|
|
}
|
|
# print(result)
|
|
return result
|
|
|
|
|
|
def get_selectors_css(files):
|
|
"""Gets the selectors and declarations from a stylesheet.
|
|
|
|
Args:
|
|
files (list): A list of path of stylesheets.
|
|
|
|
Returns:
|
|
dict: A nested dictionary with the structre as
|
|
`{'file': {'media-selector': [('selectors',`declarations')]}}`
|
|
For example:
|
|
{
|
|
'datacenterlight/css/landing-page.css':{
|
|
'(min-width: 768px)': [
|
|
('.lead-right', 'text-align: right;'),
|
|
]
|
|
}
|
|
}
|
|
"""
|
|
selectors = {}
|
|
media_selectors = {}
|
|
# get media selectors and other simple declarations
|
|
for file in files:
|
|
if any(vendor in file for vendor in ['bootstrap', 'font-awesome']):
|
|
continue
|
|
result = finders.find(file)
|
|
if result:
|
|
with open(result) as f:
|
|
data = f.read()
|
|
media_selectors[file] = string_match_pattern(data, 'css_media')
|
|
new_data = string_remove_pattern(data, 'css_media')
|
|
default_match = string_match_pattern(new_data, 'css_selector')
|
|
selectors[file] = {
|
|
'default': [
|
|
[' '.join(grp.split()) for grp in m] for m in default_match
|
|
]
|
|
}
|
|
# get declarations from media queries
|
|
for file, match_list in media_selectors.items():
|
|
for match in match_list:
|
|
query = match[0]
|
|
block_text = ' '.join(match[1].split())
|
|
results = string_match_pattern(
|
|
block_text, 'css_selector'
|
|
)
|
|
f_query = ' '.join(query.replace(':', ': ').split())
|
|
if f_query in selectors[file]:
|
|
selectors[file][f_query].extend(results)
|
|
else:
|
|
selectors[file][f_query] = results
|
|
return selectors
|
|
|
|
|
|
def get_selectors_html(files):
|
|
"""Get `class` and `id` used in html files.
|
|
|
|
Args:
|
|
files (list): A list of html files path.
|
|
|
|
Returns:
|
|
dict: a dictonary of all the classes and ids found in the file, in
|
|
`class` and `id` field respectively.
|
|
"""
|
|
selectors = {}
|
|
for file in files:
|
|
results = templates_match_pattern(file, ['html_class', 'html_id'])
|
|
class_dict = {c: 1 for match in results[0] for c in match.split()}
|
|
selectors[file] = {
|
|
'classes': list(class_dict.keys()),
|
|
'ids': results[1],
|
|
}
|
|
return selectors
|
|
|
|
|
|
def file_match_pattern(file, patterns):
|
|
"""Match a regex pattern in a file
|
|
|
|
Args:
|
|
file (str): Complete path of file
|
|
patterns (list or str): The pattern(s) to be searched in the file
|
|
|
|
Returns:
|
|
list: A list of all the matches in the file. Each item is a list of
|
|
all the captured groups in the pattern. If multiple patterns are given,
|
|
the returned list is a list of such lists.
|
|
For example:
|
|
[('.lead', 'font-size: 18px;'), ('.btn-lg', 'min-width: 180px;')]
|
|
"""
|
|
with open(file) as f:
|
|
data = f.read()
|
|
results = string_match_pattern(data, patterns)
|
|
return results
|
|
|
|
|
|
def string_match_pattern(data, patterns):
|
|
"""Match a regex pattern in a string
|
|
|
|
Args:
|
|
data (str): the string to search for the pattern
|
|
patterns (list or str): The pattern(s) to be searched in the file
|
|
|
|
Returns:
|
|
list: A list of all the matches in the string. Each item is a list of
|
|
all the captured groups in the pattern. If multiple patterns are given,
|
|
the returned list is a list of such lists.
|
|
For example:
|
|
[('.lead', 'font-size: 18px;'), ('.btn-lg', 'min-width: 180px;')]
|
|
"""
|
|
if not isinstance(patterns, str):
|
|
results = []
|
|
for p in patterns:
|
|
re_pattern = re.compile(RE_PATTERNS[p], re.MULTILINE)
|
|
results.append(re.findall(re_pattern, data))
|
|
else:
|
|
re_pattern = re.compile(RE_PATTERNS[patterns], re.MULTILINE)
|
|
results = re.findall(re_pattern, data)
|
|
return results
|
|
|
|
|
|
def string_remove_pattern(data, patterns):
|
|
"""Remove a pattern from a string
|
|
|
|
Args:
|
|
data (str): the string to search for the patter
|
|
patterns (list or str): The pattern(s) to be removed from the file
|
|
|
|
Returns:
|
|
str: The new string with all instance of matching pattern
|
|
removed from it
|
|
"""
|
|
if not isinstance(patterns, str):
|
|
for p in patterns:
|
|
re_pattern = re.compile(RE_PATTERNS[p], re.MULTILINE)
|
|
data = re.sub(re_pattern, '', data)
|
|
else:
|
|
re_pattern = re.compile(RE_PATTERNS[patterns], re.MULTILINE)
|
|
data = re.sub(re_pattern, '', data)
|
|
return data
|
|
|
|
|
|
def templates_match_pattern(template_name, patterns):
|
|
"""Match a regex pattern in the first found template file
|
|
|
|
Args:
|
|
file (str): Path of template file
|
|
patterns (list or str): The pattern(s) to be searched in the file
|
|
|
|
Returns:
|
|
list: A list of all the matches in the file. Each item is a list of
|
|
all the captured groups in the pattern. If multiple patterns are given,
|
|
the returned list is a list of such lists.
|
|
For example:
|
|
[('.lead', 'font-size: 18px;'), ('.btn-lg', 'min-width: 180px;')]
|
|
"""
|
|
t = template.loader.get_template(template_name)
|
|
data = t.template.source
|
|
results = string_match_pattern(data, patterns)
|
|
return results
|
|
|
|
|
|
def get_css_duplication(css_selectors):
|
|
"""Get duplicate selectors from the same stylesheet
|
|
|
|
Args:
|
|
css_selectors (dict): A dictonary containing css selectors from
|
|
all the files in the app in the below structure.
|
|
`{'file': {'media-selector': [('selectors',`declarations')]}}`
|
|
|
|
Returns:
|
|
dict: A dictonary containing the count of any duplicate selector in
|
|
each file.
|
|
`{'file': {'media-selector': {'selector': count}}}`
|
|
"""
|
|
# duplicate css selectors in stylesheets
|
|
rule_count = {}
|
|
for file, media_selectors in css_selectors.items():
|
|
rule_count[file] = {}
|
|
for media, rules in media_selectors.items():
|
|
rules_dict = Counter([rule[0] for rule in rules])
|
|
dup_rules_dict = {k: v for k, v in rules_dict.items() if v > 1}
|
|
if dup_rules_dict:
|
|
rule_count[file][media] = dup_rules_dict
|
|
return rule_count
|
|
|
|
|
|
def get_css_unused(css_selectors, html_selectors):
|
|
"""Get selectors from stylesheets that are not used in any of the html
|
|
files in which the stylesheet is used.
|
|
|
|
Args:
|
|
css_selectors (dict): A dictonary containing css selectors from
|
|
all the files in the app in the below structure.
|
|
`{'file': {'media-selector': [('selectors',`declarations')]}}`
|
|
html_selectors (dict): A dictonary containing the 'class' and 'id'
|
|
declarations from all html files
|
|
"""
|
|
with open('utils/optimize/test.json', 'w') as f:
|
|
json.dump([html_selectors, css_selectors], f, indent=4)
|
|
# print(html_selectors, css_selectors)
|
|
|
|
|
|
def write_report(all_reports, filename='frontend'):
|
|
"""Write the generated report to a file for re-use
|
|
|
|
Args;
|
|
all_reports (dict): A dictonary of report obtained from different tests
|
|
filename (str): An optional suffix for the output file
|
|
"""
|
|
# full_filename = 'utils/optimize/optimize_' + filename + '.html'
|
|
# output_file = os.path.join(
|
|
# settings.PROJECT_DIR, full_filename
|
|
# )
|
|
with open('utils/optimize/op_frontend.json', 'w') as f:
|
|
json.dump(all_reports, f, indent=4)
|
|
# with open(output_file, 'w', newline='') as f:
|
|
# f.write(
|
|
# template.loader.render_to_string(
|
|
# 'utils/report.html', {'all_reports': all_reports}
|
|
# )
|
|
# )
|
|
# w = csv.writer(f)
|
|
# print(zip_longest(*results))
|
|
# for r in zip_longest(*results):
|
|
# w.writerow(r)
|
|
|
|
|
|
# a list of all the html tags (to be moved in a json file)
|
|
html_tags = [
|
|
"a",
|
|
"abbr",
|
|
"address",
|
|
"article",
|
|
"area",
|
|
"aside",
|
|
"audio",
|
|
"b",
|
|
"base",
|
|
"bdi",
|
|
"bdo",
|
|
"blockquote",
|
|
"body",
|
|
"br",
|
|
"button",
|
|
"canvas",
|
|
"caption",
|
|
"cite",
|
|
"code",
|
|
"col",
|
|
"colgroup",
|
|
"datalist",
|
|
"dd",
|
|
"del",
|
|
"details",
|
|
"dfn",
|
|
"div",
|
|
"dl",
|
|
"dt",
|
|
"em",
|
|
"embed",
|
|
"fieldset",
|
|
"figcaption",
|
|
"figure",
|
|
"footer",
|
|
"form",
|
|
"h1",
|
|
"h2",
|
|
"h3",
|
|
"h4",
|
|
"h5",
|
|
"h6",
|
|
"head",
|
|
"header",
|
|
"hgroup",
|
|
"hr",
|
|
"html",
|
|
"i",
|
|
"iframe",
|
|
"img",
|
|
"input",
|
|
"ins",
|
|
"kbd",
|
|
"keygen",
|
|
"label",
|
|
"legend",
|
|
"li",
|
|
"link",
|
|
"map",
|
|
"mark",
|
|
"menu",
|
|
"meta",
|
|
"meter",
|
|
"nav",
|
|
"noscript",
|
|
"object",
|
|
"ol",
|
|
"optgroup",
|
|
"option",
|
|
"output",
|
|
"p",
|
|
"param",
|
|
"pre",
|
|
"progress",
|
|
"q",
|
|
"rp",
|
|
"rt",
|
|
"ruby",
|
|
"s",
|
|
"samp",
|
|
"script",
|
|
"section",
|
|
"select",
|
|
"source",
|
|
"small",
|
|
"span",
|
|
"strong",
|
|
"style",
|
|
"sub",
|
|
"summary",
|
|
"sup",
|
|
"textarea",
|
|
"table",
|
|
"tbody",
|
|
"td",
|
|
"tfoot",
|
|
"thead",
|
|
"th",
|
|
"time",
|
|
"title",
|
|
"tr",
|
|
"u",
|
|
"ul",
|
|
"var",
|
|
"video",
|
|
"wbr"
|
|
]
|