gmba_django/app/views.py

import csv
import json
import time

from django.contrib import messages
from werkzeug.utils import secure_filename
from django.views.generic.base import TemplateView, View
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from django.core.paginator import Paginator
from django.http import FileResponse, HttpResponsePermanentRedirect, HttpResponse, JsonResponse
from django.urls import reverse
from django.shortcuts import get_object_or_404
import operator
from django.db.models import Q
from functools import reduce
import os.path as ospath
from shutil import move

from os import makedirs
from tempfile import gettempdir
from .formats import *
from .convert import reindex_data, refresh_data
from .models import Person, RangesPeople, TaxaPeople, FieldsPeople, MethodsPeople, Range, Field, Taxon, Resource

# Get temporary file storage
UPLOAD_PATH = gettempdir()
DATA_PATH = ospath.join(ospath.dirname(__file__), '..')
if not ospath.exists(DATA_PATH):
    makedirs(DATA_PATH)


def get_datafile(fmt):
    return ospath.join(
        DATA_PATH,
        fmt['folder'],
        fmt['filename'] + '.' + fmt['extension']
    )


# Data update tracking
c_progress = 0
c_filename = ""


class HomePageView(TemplateView):
    template_name = "app/index.html"

    def get_context_data(self, **kwargs):
        fmts = DATAFORMATS
        for f in fmts:
            f['ready'] = ospath.isfile(get_datafile(f))
        context = super().get_context_data(**kwargs)
        return context


class OfflinePageView(TemplateView):
    template_name = "app/offline.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        return context


class DemoPageView(TemplateView):
    template_name = "app/demo.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        return context


class ReindexPageView(TemplateView):
    template_name = ""

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        return context

    def post(self, request, **kwargs):
        global c_progress
        c_progress = 0
        global c_filename
        c_filename = ""
        reindex_data()
        messages.add_message(request, messages.INFO, 'Search engine refresh complete')
        print("Search engine reindexed")
        return HttpResponsePermanentRedirect(reverse('admin:index'))


class RefreshPageView(TemplateView):
    template_name = ""

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        return context

    def post(self, request):
        global c_progress
        c_progress = 0

        def generate():
            stats = []
            total = 0
            for fmt in DATAFORMATS:
                global c_filename
                c_filename = fmt['filename']
                filename = get_datafile(fmt)
                c = 1
                c_counter = 0
                rd = refresh_data(filename, fmt)
                while c is not None:
                    try:
                        c, p = next(rd)
                    except Exception as e:
                        yield 'error: %s' % str(e)
                        #traceback.print_exc()
                        return
                    if isinstance(c, (int, float)):
                        global c_progress
                        c_counter = c
                        if isinstance(p, (int, float)):
                            c_progress = p
                        yield str(c) + "\n\n"
                    elif isinstance(p, str) and isinstance(c, str):
                        # Error condition
                        yield p + ": " + c + "\n\n"
                        return

                stats.append({'format': fmt['dataformat'], 'count': c_counter})
                print("Refresh: %d counted at %s" % (c_counter, fmt['dataformat']))
                total = total + c_counter

            yield "done: %d objects updated" % total
            print("Refresh: %d objects updated" % total)

            c_progress = 0
            c_filename = ""
        return HttpResponse(generate(), content_type='text/html')


class ConfigurationPageView(TemplateView):
    template_name = "app/admin/config.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        return context

    def post(self, request, **kwargs):
        global c_progress
        c_progress = 0
        global c_filename
        c_filename = ""
        reindex_data()
        messages.add_message(request, messages.INFO, 'Search engine refresh complete')
        print("Search engine reindexed")
        return HttpResponsePermanentRedirect(reverse('admin:index'))


class ConfigurationHomePageView(TemplateView):
    template_name = "app/admin/index.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        return context


def get_paginated(query_set, page, per_page):
    paginator = Paginator(query_set, per_page).page(page)
    filters = {
        'country': [],
        'range': [],
        'field': [],
        'taxon': [],
    }
    for p in paginator.object_list:
        filters['country'].append(p.country)
        for r in p.rangespeople_set.all():
            filters['range'].append(r.range.name)
        for r in p.fieldspeople_set.all():
            filters['field'].append(r.field.name)
        for r in p.taxapeople_set.all():
            filters['taxon'].append(r.taxon.name)
    filters = {
        'country': sorted(set(filters['country'])),
        'range': sorted(set(filters['range'])),
        'field': sorted(set(filters['field'])),
        'taxon': sorted(set(filters['taxon'])),
    }

    if len(query_set.all()) > len(paginator.object_list) and paginator.paginator.count == 1:
        paginator.object_list = query_set.all()
        paginator.count(len(paginator.object_list))
    return_data = {
        'items': [p.dict() for p in paginator.object_list],
        'filters': filters,
        'page': page, 'pages': paginator.paginator.num_pages, 'total': paginator.paginator.count,
        'has_next': paginator.has_next(), 'has_prev': paginator.has_previous()
    }
    return return_data


class SearchView(View):

    def get(self, request):
        page = int(self.request.GET.get('page', 1))
        per_page = int(self.request.GET.get('per_page', 10))
        q = self.request.GET.get('q', '').strip()
        if not q or len(q) < 3:
            query_set = Person.objects.all()
        else:
            query = reduce(operator.or_, (Q(field_indexer__icontains=item) for item in q.split(" ")))
            query_set = Person.objects.filter(query)

        q_country = self.request.GET.get('country', '')
        q_range = self.request.GET.get('range', '')
        q_field = self.request.GET.get('field', '')
        q_taxon = self.request.GET.get('taxon', '')
        if len(q_country) > 2:
            query_set = query_set.filter(country__icontains=q_country.strip().lower())
        if len(q_range) > 2:
            ranges_people = RangesPeople.objects.filter(range__name__icontains=q_range.strip().lower())
            r_people_ids = [rp.person_id for rp in ranges_people]
            query_set = query_set.filter(id__in=r_people_ids)
        if len(q_field) > 2:
            fields_people = FieldsPeople.objects.filter(field__name__icontains=q_field.strip().lower())
            f_people_ids = [fp.person_id for fp in fields_people]
            query_set = query_set.filter(id__in=f_people_ids)
        if len(q_taxon) > 2:
            taxa_people = TaxaPeople.objects.filter(taxon__name__icontains=q_taxon.strip().lower())
            t_people_ids = [tp.person_id for tp in taxa_people]
            query_set = query_set.filter(id__in=t_people_ids)
        query_set = query_set.order_by('last_name')
        return JsonResponse(get_paginated(query_set, page, per_page))


class PeopleDetailView(View):
    def get(self, request, people_id):
        person = get_object_or_404(Person, id=people_id)
        return_data = {
            'data': person.dict(),
            'resources': [r.dict() for r in person.resourcespeople_set.all()],
            'ranges': [r.dict() for r in person.rangespeople_set.all()],
            'fields': [r.name for r in person.fieldspeople_set.all()],
            'methods': [r.name for r in person.methodspeople_set.all()],
            'scales': [r.name for r in person.scalespeople_set.all()],
            'taxa': [r.name for r in person.taxapeople_set.all()],
        }
        return JsonResponse(return_data)


class PeopleListView(View):
    def get(self, request):
        page = int(self.request.GET.get('page', 1))
        per_page = int(self.request.GET.get('per_page', 10))
        query_set = Person.objects.all().order_by('last_name')
        return JsonResponse(get_paginated(query_set, page, per_page))


class ResourceListView(View):
    def get(self, request):
        page = int(self.request.GET.get('page', 1))
        per_page = int(self.request.GET.get('per_page', 10))
        query_set = Resource.objects.all().order_by('title')
        return JsonResponse(get_paginated(query_set, page, per_page))


MAX_FILTER_RESULTS = 50


class RangesListView(View):
    def get(self, request):
        q = self.request.GET.get('q', '').strip()
        if not q or len(q) < 3:
            query_set = Range.objects.all().order_by('name')[:MAX_FILTER_RESULTS]
        else:
            query_set = Range.objects.filter(name__icontains=q.strip().lower())
        return JsonResponse(list(query_set.values()), safe=False)


class FieldsListView(View):
    def get(self, request):
        q = self.request.GET.get('q', '').strip()
        if not q or len(q) < 3:
            query_set = Field.objects.all().order_by('name')[:MAX_FILTER_RESULTS]
        else:
            query_set = Field.objects.filter(name__icontains=q.strip().lower())
        return JsonResponse(list(query_set.values()), safe=False)


class TaxaListView(View):
    def get(self, request):
        q = self.request.GET.get('q', '').strip()
        if not q or len(q) < 3:
            query_set = Taxon.objects.all().order_by('name')[:MAX_FILTER_RESULTS]
        else:
            query_set = Taxon.objects.filter(name__icontains=q.strip().lower())
        return JsonResponse(list(query_set.values()), safe=False)


class UploadView(TemplateView):
    template_name = "app/admin/config.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        return context

    def get(self, request):
        messages.add_message(request, messages.ERROR, 'Please select a valid file')
        return HttpResponsePermanentRedirect(reverse('app:config'))

    def post(self, request, **kwargs):
        if 'datafile' in request.FILES:
            fs = request.FILES['datafile']
            fs_name = secure_filename(fs.name)
            fs_path = default_storage.save(fs_name, ContentFile(fs.read()))
            print('Uploading: %s' % fs_path)

            # Validation
            fmt = None
            if fs_name.endswith('.csv'):
                with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as csvfile:
                    datareader = csv.DictReader(csvfile)
                    datalist = list(datareader)
                    fmt = detect_dataformat(datalist[0])

            elif fs_name.endswith('.geojson'):
                with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as jsonfile:
                    jsondata = json.load(jsonfile)
                    fmt = detect_dataformat(jsondata['features'][0]['properties'])

            # Loading
            if fmt is not None:
                fs_target = get_datafile(fmt)
                move(fs_path, fs_target)
                messages.add_message(request, message="Uploaded new data file %s" % fmt['filename'],
                                     level=messages.SUCCESS)
            else:
                messages.add_message(request, message="Could not validate data format!", level=messages.ERROR)
        else:
            messages.add_message(request, message="Please select a valid file", level=messages.ERROR)
        return HttpResponsePermanentRedirect(reverse('config'))


def get_progress(request):
    global c_progress
    global c_filename

    def generate():
        while 1:
            p = str(100*c_progress)
            print("progress = %s" % p)
            yield 'data: { "p":'+p+',"f":"'+c_filename+'"}\n\n'
            time.sleep(1.0)
    if c_filename == "":
        return HttpResponse("{}", content_type='text/event-stream')
    response = HttpResponse(generate(), content_type='text/event-stream')
    response['X-Accel-Buffering'] = 'no'
    return response


def send_from_file(request, filename):
    request_for = ''
    if request.path.startswith('/geodata'):
        request_for = 'geodata'
    elif request.path.startswith('/data'):
        request_for = 'data'
    file_path = ospath.join(DATA_PATH, request_for, filename)
    file_to_send = open(file_path, 'rb')
    response = FileResponse(file_to_send)
    return response