378 lines
13 KiB
Python
378 lines
13 KiB
Python
import csv
|
|
import json
|
|
import time
|
|
|
|
from django.contrib import messages
|
|
from werkzeug.utils import secure_filename
|
|
from django.views.generic.base import TemplateView, View
|
|
from django.core.files.storage import default_storage
|
|
from django.core.files.base import ContentFile
|
|
from django.core.paginator import Paginator
|
|
from django.http import FileResponse, HttpResponsePermanentRedirect, HttpResponse, JsonResponse
|
|
from django.urls import reverse
|
|
from django.shortcuts import get_object_or_404
|
|
import operator
|
|
from django.db.models import Q
|
|
from functools import reduce
|
|
import os.path as ospath
|
|
from shutil import move
|
|
|
|
from os import makedirs
|
|
from tempfile import gettempdir
|
|
from .formats import *
|
|
from .convert import reindex_data, refresh_data
|
|
from .models import Person, PeopleRange, TaxaPeople, FieldsPeople, MethodsPeople, MountainRange, Field, Taxon, Resource
|
|
|
|
# Get temporary file storage
|
|
UPLOAD_PATH = gettempdir()
|
|
DATA_PATH = ospath.join(ospath.dirname(__file__), '..')
|
|
if not ospath.exists(DATA_PATH):
|
|
makedirs(DATA_PATH)
|
|
|
|
|
|
def get_datafile(fmt):
|
|
return ospath.join(
|
|
DATA_PATH,
|
|
fmt['folder'],
|
|
fmt['filename'] + '.' + fmt['extension']
|
|
)
|
|
|
|
|
|
# Data update tracking
|
|
c_progress = 0
|
|
c_filename = ""
|
|
|
|
|
|
class HomePageView(TemplateView):
|
|
template_name = "app/index.html"
|
|
|
|
def get_context_data(self, **kwargs):
|
|
fmts = DATAFORMATS
|
|
for f in fmts:
|
|
f['ready'] = ospath.isfile(get_datafile(f))
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
|
|
class OfflinePageView(TemplateView):
|
|
template_name = "app/offline.html"
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
|
|
class DemoPageView(TemplateView):
|
|
template_name = "app/demo.html"
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
|
|
class ReindexPageView(TemplateView):
|
|
template_name = ""
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
def post(self, request, **kwargs):
|
|
global c_progress
|
|
c_progress = 0
|
|
global c_filename
|
|
c_filename = ""
|
|
reindex_data()
|
|
messages.add_message(request, messages.INFO, 'Search engine refresh complete')
|
|
print("Search engine reindexed")
|
|
return HttpResponsePermanentRedirect(reverse('admin:index'))
|
|
|
|
|
|
class RefreshPageView(TemplateView):
|
|
template_name = ""
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
def post(self, request):
|
|
global c_progress
|
|
c_progress = 0
|
|
|
|
def generate():
|
|
stats = []
|
|
total = 0
|
|
for fmt in DATAFORMATS:
|
|
global c_filename
|
|
c_filename = fmt['filename']
|
|
filename = get_datafile(fmt)
|
|
c = 1
|
|
c_counter = 0
|
|
rd = refresh_data(filename, fmt)
|
|
while c is not None:
|
|
try:
|
|
c, p = next(rd)
|
|
except Exception as e:
|
|
yield 'error: %s' % str(e)
|
|
#traceback.print_exc()
|
|
return
|
|
if isinstance(c, (int, float)):
|
|
global c_progress
|
|
c_counter = c
|
|
if isinstance(p, (int, float)):
|
|
c_progress = p
|
|
yield str(c) + "\n\n"
|
|
elif isinstance(p, str) and isinstance(c, str):
|
|
# Error condition
|
|
yield p + ": " + c + "\n\n"
|
|
return
|
|
|
|
stats.append({'format': fmt['dataformat'], 'count': c_counter})
|
|
print("Refresh: %d counted at %s" % (c_counter, fmt['dataformat']))
|
|
total = total + c_counter
|
|
|
|
yield "done: %d objects updated" % total
|
|
print("Refresh: %d objects updated" % total)
|
|
|
|
c_progress = 0
|
|
c_filename = ""
|
|
return HttpResponse(generate(), content_type='text/html')
|
|
|
|
|
|
class ConfigurationPageView(TemplateView):
|
|
template_name = "app/admin/config.html"
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
def post(self, request, **kwargs):
|
|
global c_progress
|
|
c_progress = 0
|
|
global c_filename
|
|
c_filename = ""
|
|
reindex_data()
|
|
messages.add_message(request, messages.INFO, 'Search engine refresh complete')
|
|
print("Search engine reindexed")
|
|
return HttpResponsePermanentRedirect(reverse('admin:index'))
|
|
|
|
|
|
class ConfigurationHomePageView(TemplateView):
|
|
template_name = "app/admin/index.html"
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
|
|
def get_paginated(query_set, page, per_page):
|
|
paginator = Paginator(query_set, per_page).page(page)
|
|
filters = {
|
|
'country': [],
|
|
'range': [],
|
|
'field': [],
|
|
'taxon': [],
|
|
}
|
|
for p in paginator.object_list:
|
|
filters['country'].append(p.country.short_name)
|
|
for r in p.peoplerange_set.all():
|
|
if r.range and r.range.range_name:
|
|
filters['range'].append(r.range.range_name)
|
|
else:
|
|
print("r.range.range_name is None %s %s" % (r.range_id, r.person_id))
|
|
for r in p.fieldspeople_set.all():
|
|
filters['field'].append(r.field.name)
|
|
for r in p.taxapeople_set.all():
|
|
if r and r.taxon and r.taxon.name:
|
|
filters['taxon'].append(r.taxon.name)
|
|
else:
|
|
print("r.taxon.name is None %s %s" % (r.range_id, r.taxon))
|
|
filters = {
|
|
'country': sorted(set(filters['country'])),
|
|
'range': sorted(set(filters['range'])),
|
|
'field': sorted(set(filters['field'])),
|
|
'taxon': sorted(set(filters['taxon'])),
|
|
}
|
|
|
|
if len(query_set.all()) > len(paginator.object_list) and paginator.paginator.count == 1:
|
|
paginator.object_list = query_set.all()
|
|
paginator.count(len(paginator.object_list))
|
|
return_data = {
|
|
'items': [p.dict() for p in paginator.object_list],
|
|
'filters': filters,
|
|
'page': page, 'pages': paginator.paginator.num_pages, 'total': paginator.paginator.count,
|
|
'has_next': paginator.has_next(), 'has_prev': paginator.has_previous()
|
|
}
|
|
return return_data
|
|
|
|
|
|
class SearchView(View):
|
|
|
|
def get(self, request):
|
|
page = int(self.request.GET.get('page', 1))
|
|
per_page = int(self.request.GET.get('per_page', 10))
|
|
q = self.request.GET.get('q', '').strip()
|
|
if not q or len(q) < 3:
|
|
query_set = Person.objects.filter(profile_on_web=True).all()
|
|
else:
|
|
query = reduce(operator.or_, (Q(field_indexer__icontains=item) for item in q.split(" ")))
|
|
query_set = Person.objects.filter(query).filter(profile_on_web=True)
|
|
|
|
q_country = self.request.GET.get('country', '')
|
|
q_range = self.request.GET.get('range', '')
|
|
q_field = self.request.GET.get('field', '')
|
|
q_taxon = self.request.GET.get('taxon', '')
|
|
if len(q_country) > 2:
|
|
query_set = query_set.filter(country__short_name__icontains=q_country.strip().lower())
|
|
if len(q_range) > 2:
|
|
ranges_people = PeopleRange.objects.filter(range__range_name__icontains=q_range.strip().lower())
|
|
r_people_ids = [rp.person_id for rp in ranges_people]
|
|
query_set = query_set.filter(id__in=r_people_ids)
|
|
if len(q_field) > 2:
|
|
fields_people = FieldsPeople.objects.filter(field__name__icontains=q_field.strip().lower())
|
|
f_people_ids = [fp.person_id for fp in fields_people]
|
|
query_set = query_set.filter(id__in=f_people_ids)
|
|
if len(q_taxon) > 2:
|
|
taxa_people = TaxaPeople.objects.filter(taxon__name__icontains=q_taxon.strip().lower())
|
|
t_people_ids = [tp.person_id for tp in taxa_people]
|
|
query_set = query_set.filter(id__in=t_people_ids)
|
|
query_set = query_set.order_by('last_name')
|
|
return JsonResponse(get_paginated(query_set, page, per_page))
|
|
|
|
|
|
class PeopleDetailView(View):
|
|
def get(self, request, people_id):
|
|
person = get_object_or_404(Person, id=people_id)
|
|
return_data = {
|
|
'data': person.dict(),
|
|
'resources': [r.resource.dict() for r in person.peopleresource_set.all()],
|
|
'ranges': [r.range.dict() for r in person.peoplerange_set.all()],
|
|
'fields': [r.field.name for r in person.fieldspeople_set.all()],
|
|
'methods': [r.method.name for r in person.methodspeople_set.all()],
|
|
'scales': [r.scale.name for r in person.scalespeople_set.all()],
|
|
'taxa': [r.taxon.name for r in person.taxapeople_set.all()],
|
|
}
|
|
return JsonResponse(return_data)
|
|
|
|
|
|
class PeopleListView(View):
|
|
def get(self, request):
|
|
page = int(self.request.GET.get('page', 1))
|
|
per_page = int(self.request.GET.get('per_page', 10))
|
|
query_set = Person.objects.all().order_by('last_name')
|
|
return JsonResponse(get_paginated(query_set, page, per_page))
|
|
|
|
|
|
class ResourceListView(View):
|
|
def get(self, request):
|
|
page = int(self.request.GET.get('page', 1))
|
|
per_page = int(self.request.GET.get('per_page', 10))
|
|
query_set = Resource.objects.all().order_by('title')
|
|
return JsonResponse(get_paginated(query_set, page, per_page))
|
|
|
|
|
|
MAX_FILTER_RESULTS = 50
|
|
|
|
|
|
class RangesListView(View):
|
|
def get(self, request):
|
|
q = self.request.GET.get('q', '').strip()
|
|
if not q or len(q) < 3:
|
|
query_set = MountainRange.objects.all().order_by('name')[:MAX_FILTER_RESULTS]
|
|
else:
|
|
query_set = MountainRange.objects.filter(name__icontains=q.strip().lower())
|
|
return JsonResponse(list(query_set.values()), safe=False)
|
|
|
|
|
|
class FieldsListView(View):
|
|
def get(self, request):
|
|
q = self.request.GET.get('q', '').strip()
|
|
if not q or len(q) < 3:
|
|
query_set = Field.objects.all().order_by('name')[:MAX_FILTER_RESULTS]
|
|
else:
|
|
query_set = Field.objects.filter(name__icontains=q.strip().lower())
|
|
return JsonResponse(list(query_set.values()), safe=False)
|
|
|
|
|
|
class TaxaListView(View):
|
|
def get(self, request):
|
|
q = self.request.GET.get('q', '').strip()
|
|
if not q or len(q) < 3:
|
|
query_set = Taxon.objects.all().order_by('name')[:MAX_FILTER_RESULTS]
|
|
else:
|
|
query_set = Taxon.objects.filter(name__icontains=q.strip().lower())
|
|
return JsonResponse(list(query_set.values()), safe=False)
|
|
|
|
|
|
class UploadView(TemplateView):
|
|
template_name = "app/admin/config.html"
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
return context
|
|
|
|
def get(self, request):
|
|
messages.add_message(request, messages.ERROR, 'Please select a valid file')
|
|
return HttpResponsePermanentRedirect(reverse('app:config'))
|
|
|
|
def post(self, request, **kwargs):
|
|
if 'datafile' in request.FILES:
|
|
fs = request.FILES['datafile']
|
|
fs_name = secure_filename(fs.name)
|
|
fs_path = default_storage.save(fs_name, ContentFile(fs.read()))
|
|
print('Uploading: %s' % fs_path)
|
|
|
|
# Validation
|
|
fmt = None
|
|
if fs_name.endswith('.csv'):
|
|
with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as csvfile:
|
|
datareader = csv.DictReader(csvfile)
|
|
datalist = list(datareader)
|
|
fmt = detect_dataformat(datalist[0])
|
|
|
|
elif fs_name.endswith('.geojson'):
|
|
with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as jsonfile:
|
|
jsondata = json.load(jsonfile)
|
|
fmt = detect_dataformat(jsondata['features'][0]['properties'])
|
|
|
|
# Loading
|
|
if fmt is not None:
|
|
fs_target = get_datafile(fmt)
|
|
move(fs_path, fs_target)
|
|
messages.add_message(request, message="Uploaded new data file %s" % fmt['filename'],
|
|
level=messages.SUCCESS)
|
|
else:
|
|
messages.add_message(request, message="Could not validate data format!", level=messages.ERROR)
|
|
else:
|
|
messages.add_message(request, message="Please select a valid file", level=messages.ERROR)
|
|
return HttpResponsePermanentRedirect(reverse('config'))
|
|
|
|
|
|
def get_progress(request):
|
|
global c_progress
|
|
global c_filename
|
|
|
|
def generate():
|
|
while 1:
|
|
p = str(100*c_progress)
|
|
print("progress = %s" % p)
|
|
yield 'data: { "p":'+p+',"f":"'+c_filename+'"}\n\n'
|
|
time.sleep(1.0)
|
|
if c_filename == "":
|
|
return HttpResponse("{}", content_type='text/event-stream')
|
|
response = HttpResponse(generate(), content_type='text/event-stream')
|
|
response['X-Accel-Buffering'] = 'no'
|
|
return response
|
|
|
|
|
|
def send_from_file(request, filename):
|
|
request_for = ''
|
|
if request.path.startswith('/geodata'):
|
|
request_for = 'geodata'
|
|
elif request.path.startswith('/data'):
|
|
request_for = 'data'
|
|
file_path = ospath.join(DATA_PATH, request_for, filename)
|
|
file_to_send = open(file_path, 'rb')
|
|
response = FileResponse(file_to_send)
|
|
return response
|