import csv import json import os import time from django.contrib import messages from werkzeug.utils import secure_filename from django.views.generic.base import TemplateView from django.core.files.storage import default_storage from django.core.files.base import ContentFile from django.views.decorators.csrf import csrf_protect from django.http import FileResponse, HttpResponsePermanentRedirect, HttpResponse from django.urls import reverse import os.path as ospath from shutil import move from os import makedirs from tempfile import gettempdir from .formats import * from .convert import reindex_data, refresh_data # Get temporary file storage UPLOAD_PATH = gettempdir() DATA_PATH = ospath.join(ospath.dirname(__file__), '..') if not ospath.exists(DATA_PATH): makedirs(DATA_PATH) def get_datafile(fmt): return ospath.join( DATA_PATH, fmt['folder'], fmt['filename'] + '.' + fmt['extension'] ) # Data update tracking c_progress = 0 c_filename = "" class HomePageView(TemplateView): template_name = "app/index.html" def get_context_data(self, **kwargs): fmts = DATAFORMATS for f in fmts: f['ready'] = ospath.isfile(get_datafile(f)) context = super().get_context_data(**kwargs) return context class OfflinePageView(TemplateView): template_name = "app/offline.html" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) return context class DemoPageView(TemplateView): template_name = "app/demo.html" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) return context class ReindexPageView(TemplateView): template_name = "app/demo.html" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) return context def post(self, request, **kwargs): global c_progress c_progress = 0 global c_filename c_filename = "" reindex_data() messages.add_message(request, messages.INFO, 'Search engine refresh complete') print("Search engine reindexed") return HttpResponsePermanentRedirect(reverse('admin:index')) class RefreshPageView(TemplateView): template_name = "" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) return context def post(self, request): global c_progress c_progress = 0 def generate(): stats = [] total = 0 for fmt in DATAFORMATS: global c_filename c_filename = fmt['filename'] filename = get_datafile(fmt) c = 1 c_counter = 0 rd = refresh_data(filename, fmt) while c is not None: try: c, p = next(rd) except Exception as e: yield 'error: %s' % str(e) #traceback.print_exc() return if isinstance(c, (int, float)): global c_progress c_counter = c if isinstance(p, (int, float)): c_progress = p yield str(c) + "\n\n" elif isinstance(p, str) and isinstance(c, str): # Error condition yield p + ": " + c + "\n\n" return stats.append({'format': fmt['dataformat'], 'count': c_counter}) print("Refresh: %d counted at %s" % (c_counter, fmt['dataformat'])) total = total + c_counter yield "done: %d objects updated" % total print("Refresh: %d objects updated" % total) c_progress = 0 c_filename = "" return HttpResponse(generate(), content_type='text/html') class ConfigurationPageView(TemplateView): template_name = "app/admin/config.html" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) return context def post(self, request, **kwargs): global c_progress c_progress = 0 global c_filename c_filename = "" reindex_data() messages.add_message(request, messages.INFO, 'Search engine refresh complete') print("Search engine reindexed") return HttpResponsePermanentRedirect(reverse('admin:index')) class UploadView(TemplateView): template_name = "app/admin/config.html" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) return context def get(self, request): messages.add_message(request, messages.ERROR, 'Please select a valid file') return HttpResponsePermanentRedirect(reverse('app:config')) def post(self, request, **kwargs): if 'datafile' in request.FILES: fs = request.FILES['datafile'] fs_name = secure_filename(fs.name) fs_path = default_storage.save(fs_name, ContentFile(fs.read())) print('Uploading: %s' % fs_path) # Validation fmt = None if fs_name.endswith('.csv'): with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as csvfile: datareader = csv.DictReader(csvfile) datalist = list(datareader) fmt = detect_dataformat(datalist[0]) elif fs_name.endswith('.geojson'): with open(fs_path, 'rt', encoding='utf-8', errors='ignore') as jsonfile: jsondata = json.load(jsonfile) fmt = detect_dataformat(jsondata['features'][0]['properties']) # Loading if fmt is not None: fs_target = get_datafile(fmt) move(fs_path, fs_target) messages.add_message(request, message="Uploaded new data file %s" % fmt['filename'], level=messages.SUCCESS) else: messages.add_message(request, message="Could not validate data format!", level=messages.ERROR) else: messages.add_message(request, message="Please select a valid file", level=messages.ERROR) return HttpResponsePermanentRedirect(reverse('config')) def get_progress(request): global c_progress global c_filename def generate(): while 1: p = str(100*c_progress) yield 'data: { "p":'+p+',"f":"'+c_filename+'"}\n\n' time.sleep(1.0) if c_filename == "": return HttpResponse("{}", content_type='text/event-stream') response = HttpResponse(generate(), content_type='text/event-stream') response['X-Accel-Buffering'] = 'no' return response def send_from_file(request, filename): request_for = '' if request.path.startswith('/geodata'): request_for = 'geodata' elif request.path.startswith('/data'): request_for = 'data' file_path = ospath.join(DATA_PATH, request_for, filename) file_to_send = open(file_path, 'rb') response = FileResponse(file_to_send) return response