gmba_django/app/views.py
2021-08-31 20:22:41 +05:30

297 lines
10 KiB
Python

import csv
import json
import time
from django.contrib import messages
from werkzeug.utils import secure_filename
from django.views.generic.base import TemplateView, View
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from django.core.paginator import Paginator
from django.http import FileResponse, HttpResponsePermanentRedirect, HttpResponse, JsonResponse
from django.urls import reverse
import os.path as ospath
from shutil import move
from os import makedirs
from tempfile import gettempdir
from .formats import *
from .convert import reindex_data, refresh_data
from .models import Person, RangesPeople, TaxaPeople, FieldsPeople, MethodsPeople, Range
# Get temporary file storage
UPLOAD_PATH = gettempdir()
DATA_PATH = ospath.join(ospath.dirname(__file__), '..')
if not ospath.exists(DATA_PATH):
makedirs(DATA_PATH)
def get_datafile(fmt):
return ospath.join(
DATA_PATH,
fmt['folder'],
fmt['filename'] + '.' + fmt['extension']
)
# Data update tracking
c_progress = 0
c_filename = ""
class HomePageView(TemplateView):
template_name = "app/index.html"
def get_context_data(self, **kwargs):
fmts = DATAFORMATS
for f in fmts:
f['ready'] = ospath.isfile(get_datafile(f))
context = super().get_context_data(**kwargs)
return context
class OfflinePageView(TemplateView):
template_name = "app/offline.html"
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
return context
class DemoPageView(TemplateView):
template_name = "app/demo.html"
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
return context
class ReindexPageView(TemplateView):
template_name = ""
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
return context
def post(self, request, **kwargs):
global c_progress
c_progress = 0
global c_filename
c_filename = ""
reindex_data()
messages.add_message(request, messages.INFO, 'Search engine refresh complete')
print("Search engine reindexed")
return HttpResponsePermanentRedirect(reverse('admin:index'))
class RefreshPageView(TemplateView):
template_name = ""
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
return context
def post(self, request):
global c_progress
c_progress = 0
def generate():
stats = []
total = 0
for fmt in DATAFORMATS:
global c_filename
c_filename = fmt['filename']
filename = get_datafile(fmt)
c = 1
c_counter = 0
rd = refresh_data(filename, fmt)
while c is not None:
try:
c, p = next(rd)
except Exception as e:
yield 'error: %s' % str(e)
#traceback.print_exc()
return
if isinstance(c, (int, float)):
global c_progress
c_counter = c
if isinstance(p, (int, float)):
c_progress = p
yield str(c) + "\n\n"
elif isinstance(p, str) and isinstance(c, str):
# Error condition
yield p + ": " + c + "\n\n"
return
stats.append({'format': fmt['dataformat'], 'count': c_counter})
print("Refresh: %d counted at %s" % (c_counter, fmt['dataformat']))
total = total + c_counter
yield "done: %d objects updated" % total
print("Refresh: %d objects updated" % total)
c_progress = 0
c_filename = ""
return HttpResponse(generate(), content_type='text/html')
class ConfigurationPageView(TemplateView):
template_name = "app/admin/config.html"
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
return context
def post(self, request, **kwargs):
global c_progress
c_progress = 0
global c_filename
c_filename = ""
reindex_data()
messages.add_message(request, messages.INFO, 'Search engine refresh complete')
print("Search engine reindexed")
return HttpResponsePermanentRedirect(reverse('admin:index'))
class ConfigurationHomePageView(TemplateView):
template_name = "app/admin/index.html"
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
return context
class SearchView(View):
def get(self, request):
page = int(self.request.GET.get('page', 1))
per_page = int(self.request.GET.get('per_page', 10))
q = self.request.GET.get('q', '').strip()
if not q or len(q) < 3:
query_set = Person.objects.all()
else:
query_set = Person.objects.filter(field_indexer__icontains=q.split(" "))
q_country = self.request.GET.get('country', '')
q_range = self.request.GET.get('range', '')
q_field = self.request.GET.get('field', '')
q_taxon = self.request.GET.get('taxon', '')
if len(q_country) > 2:
query_set = query_set.filter(country__icontains=q_country.strip().lower())
if len(q_range) > 2:
ranges_people = RangesPeople.objects.filter(range__name__icontains=q_range.strip().lower())
r_people_ids = [rp.person_id for rp in ranges_people]
query_set = query_set.filter(id__in=r_people_ids)
if len(q_field) > 2:
fields_people = FieldsPeople.objects.filter(field__name__icontains=q_field.strip().lower())
f_people_ids = [fp.person_id for fp in fields_people]
query_set = query_set.filter(id__in=f_people_ids)
if len(q_taxon) > 2:
taxa_people = TaxaPeople.objects.filter(taxon__name__icontains=q_taxon.strip().lower())
t_people_ids = [tp.person_id for tp in taxa_people]
query_set = query_set.filter(id__in=t_people_ids)
query_set = query_set.order_by('last_name')
paginator = Paginator(query_set, per_page).page(page)
filters = {
'country': [],
'range': [],
'field': [],
'taxon': [],
}
for p in paginator.object_list:
filters['country'].append(p.country)
for r in p.rangespeople_set.all():
filters['range'].append(r.name)
for r in p.fieldspeople_set.all():
filters['field'].append(r.name)
for r in p.taxapeople_set.all():
filters['taxon'].append(r.name)
filters = {
'country': sorted(set(filters['country'])),
'range': sorted(set(filters['range'])),
'field': sorted(set(filters['field'])),
'taxon': sorted(set(filters['taxon'])),
}
if len(query_set.all()) > len(paginator.object_list) and paginator.paginator.count == 1:
paginator.object_list = query_set.all()
paginator.count(len(paginator.object_list))
return_data = {
'items': [p.dict() for p in paginator.object_list],
'filters': filters,
'page': page, 'pages': paginator.paginator.num_pages, 'total': paginator.paginator.count,
'has_next': paginator.has_next(), 'has_prev': paginator.has_previous()
}
return JsonResponse(return_data)
class UploadView(TemplateView):
template_name = "app/admin/config.html"
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
return context
def get(self, request):
messages.add_message(request, messages.ERROR, 'Please select a valid file')
return HttpResponsePermanentRedirect(reverse('app:config'))
def post(self, request, **kwargs):
if 'datafile' in request.FILES:
fs = request.FILES['datafile']
fs_name = secure_filename(fs.name)
fs_path = default_storage.save(fs_name, ContentFile(fs.read()))
print('Uploading: %s' % fs_path)
# Validation
fmt = None
if fs_name.endswith('.csv'):
with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as csvfile:
datareader = csv.DictReader(csvfile)
datalist = list(datareader)
fmt = detect_dataformat(datalist[0])
elif fs_name.endswith('.geojson'):
with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as jsonfile:
jsondata = json.load(jsonfile)
fmt = detect_dataformat(jsondata['features'][0]['properties'])
# Loading
if fmt is not None:
fs_target = get_datafile(fmt)
move(fs_path, fs_target)
messages.add_message(request, message="Uploaded new data file %s" % fmt['filename'],
level=messages.SUCCESS)
else:
messages.add_message(request, message="Could not validate data format!", level=messages.ERROR)
else:
messages.add_message(request, message="Please select a valid file", level=messages.ERROR)
return HttpResponsePermanentRedirect(reverse('config'))
def get_progress(request):
global c_progress
global c_filename
def generate():
while 1:
p = str(100*c_progress)
yield 'data: { "p":'+p+',"f":"'+c_filename+'"}\n\n'
time.sleep(1.0)
if c_filename == "":
return HttpResponse("{}", content_type='text/event-stream')
response = HttpResponse(generate(), content_type='text/event-stream')
response['X-Accel-Buffering'] = 'no'
return response
def send_from_file(request, filename):
request_for = ''
if request.path.startswith('/geodata'):
request_for = 'geodata'
elif request.path.startswith('/data'):
request_for = 'data'
file_path = ospath.join(DATA_PATH, request_for, filename)
file_to_send = open(file_path, 'rb')
response = FileResponse(file_to_send)
return response