Merge pull request 'fix_refresh_data' (#5) from fix_refresh_data into master
Reviewed-on: #5
This commit is contained in:
commit
907ae0c256
3 changed files with 96 additions and 34 deletions
|
@ -71,7 +71,8 @@ def reindex_data():
|
||||||
|
|
||||||
|
|
||||||
# Data update routine
|
# Data update routine
|
||||||
def refresh_data(filename, fmt=None):
|
def refresh_data(filename, fmt=None, update_existing=False):
|
||||||
|
print("refresh_data")
|
||||||
count = 0
|
count = 0
|
||||||
rowcount = 0
|
rowcount = 0
|
||||||
if not isfile(filename):
|
if not isfile(filename):
|
||||||
|
@ -101,18 +102,22 @@ def refresh_data(filename, fmt=None):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if fmt['dataformat'] is DataFormat.PERSON_DETAIL:
|
if fmt['dataformat'] is DataFormat.PERSON_DETAIL:
|
||||||
|
print(row)
|
||||||
person, source_id = get_by_id(row['ID'], Person)
|
person, source_id = get_by_id(row['ID'], Person)
|
||||||
if not person:
|
if not person:
|
||||||
person = Person.objects.filter(first_name=row['First name'], last_name=row['Last name']).first()
|
person = Person.objects.filter(first_name=row['First name'], last_name=row['Last name']).first()
|
||||||
|
if person:
|
||||||
|
print("Fetched from DB")
|
||||||
|
else:
|
||||||
|
print("Does not exist in DB")
|
||||||
if not person:
|
if not person:
|
||||||
person = Person(first_name=row['First name'], last_name=row['Last name'], source_id=row['ID'])
|
person = Person(first_name=row['First name'], last_name=row['Last name'], source_id=row['ID'])
|
||||||
|
|
||||||
# Update data fields
|
|
||||||
person.source_id = source_id
|
person.source_id = source_id
|
||||||
person.title = row['Title']
|
person.title = row['Title']
|
||||||
|
print(row)
|
||||||
person.organisation = row['Organisation English']
|
person.organisation = row['Organisation English']
|
||||||
print("Country = %s" % row['country'])
|
print("Country = %s" % row['country'] if 'country' in row else '')
|
||||||
if row['country'] is None or row['country'].strip() == '':
|
if 'country' not in row or row['country'] is None or row['country'].strip() == '':
|
||||||
row['country'] = 0
|
row['country'] = 0
|
||||||
c = Country.objects.get(id=row['country'])
|
c = Country.objects.get(id=row['country'])
|
||||||
person.country = c
|
person.country = c
|
||||||
|
@ -120,23 +125,49 @@ def refresh_data(filename, fmt=None):
|
||||||
person.biography = row['Biography']
|
person.biography = row['Biography']
|
||||||
person.contact_email = row['e-mail 1']
|
person.contact_email = row['e-mail 1']
|
||||||
person.personal_url = fix_url(row['URL'])
|
person.personal_url = fix_url(row['URL'])
|
||||||
|
person.save()
|
||||||
|
print("Created")
|
||||||
|
|
||||||
|
# Update data fields
|
||||||
|
if update_existing:
|
||||||
|
person.source_id = source_id
|
||||||
|
person.title = row['Title']
|
||||||
|
print(row)
|
||||||
|
person.organisation = row['Organisation English']
|
||||||
|
print("Country = %s" % row['country'] if 'country' in row else '')
|
||||||
|
if 'country' not in row or row['country'] is None or row['country'].strip() == '':
|
||||||
|
row['country'] = 0
|
||||||
|
c = Country.objects.get(id=row['country'])
|
||||||
|
person.country = c
|
||||||
|
person.position = row['Position']
|
||||||
|
person.biography = row['Biography']
|
||||||
|
person.contact_email = row['e-mail 1']
|
||||||
|
person.personal_url = fix_url(row['URL'])
|
||||||
|
person.save()
|
||||||
with transaction.atomic():
|
with transaction.atomic():
|
||||||
research_methods = add_linked(person, Method, row['Methods'])
|
research_methods = add_linked(person, Method, row['Methods'])
|
||||||
methods_people = [MethodsPeople.objects.get_or_create(method_id=m.id, person_id=person.id) for m in research_methods]
|
methods_people = [MethodsPeople.objects.get_or_create(method_id=m.id, person_id=person.id)[0] for m in research_methods]
|
||||||
|
person.methodspeople_set.set(methods_people)
|
||||||
research_scales = add_linked(person, Scale, row['Scale'])
|
research_scales = add_linked(person, Scale, row['Scale'])
|
||||||
scales_people = [ScalesPeople.objects.get_or_create(scale_id=s.id, person_id=person.id) for s in research_scales]
|
scales_people = [ScalesPeople.objects.get_or_create(scale_id=s.id, person_id=person.id)[0] for s in research_scales]
|
||||||
|
person.scalespeople_set.set(scales_people)
|
||||||
research_taxa = add_linked(person, Taxon, row['Taxa'])
|
research_taxa = add_linked(person, Taxon, row['Taxa'])
|
||||||
taxa_people = [TaxaPeople.objects.get_or_create(taxon_id=t.id, person_id=person.id) for t in research_taxa]
|
taxa_people = [TaxaPeople.objects.get_or_create(taxon_id=t.id, person_id=person.id)[0] for t in research_taxa]
|
||||||
|
person.taxapeople_set.set(taxa_people)
|
||||||
research_fields = add_linked(person, Field, row['Field of expertise'])
|
research_fields = add_linked(person, Field, row['Field of expertise'])
|
||||||
fields_people = [FieldsPeople.objects.get_or_create(field_id=f.id, person_id=person.id) for f in research_fields]
|
fields_people = [FieldsPeople.objects.get_or_create(field_id=f.id, person_id=person.id)[0] for f in research_fields]
|
||||||
|
person.fieldspeople_set.set(fields_people)
|
||||||
|
# research_ranges = add_linked(person, MountainRange, row['MountainRange'])
|
||||||
|
# ranges_people = [RangesPeople.objects.get_or_create(range_id=r.id, person_id=person.id)[0] for r in research_ranges]
|
||||||
|
# person.rangespeople_set.set(ranges_people)
|
||||||
person.index()
|
person.index()
|
||||||
person.save()
|
person.save()
|
||||||
count = count + 1
|
count = count + 1
|
||||||
|
|
||||||
elif fmt['dataformat'] is DataFormat.RESOURCE_DETAIL:
|
elif fmt['dataformat'] is DataFormat.RESOURCE_DETAIL:
|
||||||
res, source_id = get_by_id(row['ID'], Resource)
|
res, source_id = get_by_id(row['ID'], Resource)
|
||||||
if not res: res = Resource(source_id=source_id)
|
if not res:
|
||||||
|
res = Resource(source_id=source_id)
|
||||||
res.title = row['Title']
|
res.title = row['Title']
|
||||||
res.citation = row['Citation']
|
res.citation = row['Citation']
|
||||||
res.url = fix_url(row['URL'].strip('#')) # remove weird #formatting#
|
res.url = fix_url(row['URL'].strip('#')) # remove weird #formatting#
|
||||||
|
@ -155,9 +186,11 @@ def refresh_data(filename, fmt=None):
|
||||||
|
|
||||||
elif fmt['dataformat'] is DataFormat.PERSON_RESOURCE:
|
elif fmt['dataformat'] is DataFormat.PERSON_RESOURCE:
|
||||||
rzs, source_id = get_by_id(row['Resource'], Resource, first=False)
|
rzs, source_id = get_by_id(row['Resource'], Resource, first=False)
|
||||||
if not rzs or not rzs.first(): continue
|
if not rzs or not rzs.first():
|
||||||
|
continue
|
||||||
ppl, source_id = get_by_id(row['Person'], Person, first=False)
|
ppl, source_id = get_by_id(row['Person'], Person, first=False)
|
||||||
if not ppl or not ppl.first(): continue
|
if not ppl or not ppl.first():
|
||||||
|
continue
|
||||||
for person in ppl:
|
for person in ppl:
|
||||||
person.resources = []
|
person.resources = []
|
||||||
for r in rzs:
|
for r in rzs:
|
||||||
|
@ -167,15 +200,30 @@ def refresh_data(filename, fmt=None):
|
||||||
|
|
||||||
elif fmt['dataformat'] is DataFormat.PERSON_RANGE:
|
elif fmt['dataformat'] is DataFormat.PERSON_RANGE:
|
||||||
rzs, source_id = get_by_id(row['MountainRange'], MountainRange, first=False)
|
rzs, source_id = get_by_id(row['MountainRange'], MountainRange, first=False)
|
||||||
|
print(" range=%s, source_id=%s" % (rzs, source_id))
|
||||||
if not rzs or not rzs.first():
|
if not rzs or not rzs.first():
|
||||||
|
print(" --- No rzs, continue")
|
||||||
continue
|
continue
|
||||||
ppl, source_id = get_by_id(row['Person'], Person, first=False)
|
ppl, source_id = get_by_id(row['Person'], Person, first=False)
|
||||||
if not ppl or not ppl.first(): continue
|
print(" +++ ppl=%s, source_id=%s" % (ppl, source_id))
|
||||||
for person in ppl:
|
if not ppl or not ppl.first():
|
||||||
person.ranges = []
|
print(" --- No ppl, continue")
|
||||||
for r in rzs:
|
continue
|
||||||
person.ranges.append(r)
|
with transaction.atomic():
|
||||||
|
person = ppl.first()
|
||||||
|
research_ranges = add_linked(person, MountainRange, row['MountainRange'])
|
||||||
|
ranges_people = [PeopleRange.objects.get_or_create(range_id=r.id, person_id=person.id)[0] for r in research_ranges]
|
||||||
|
person.peoplerange_set.set(ranges_people)
|
||||||
|
# for person in ppl:
|
||||||
|
# research_ranges = add_linked(person, MountainRange, row['MountainRange'])
|
||||||
|
# ranges_people = [RangesPeople.objects.get_or_create(range_id=r.id, person_id=person.id)[0] for r in research_ranges]
|
||||||
|
# person.rangespeople_set.set(ranges_people)
|
||||||
|
# ranges_people = RangesPeople
|
||||||
|
# for r in rzs:
|
||||||
|
# ranges_people.append(r)
|
||||||
|
# person.rangespeople_set.set(ranges_people)
|
||||||
person.save()
|
person.save()
|
||||||
|
#print(" *** Saved %s => %s (%s)" % (person, ranges_people, len(ranges_people)))
|
||||||
count = count + 1
|
count = count + 1
|
||||||
|
|
||||||
elif fmt['extension'] == 'geojson':
|
elif fmt['extension'] == 'geojson':
|
||||||
|
|
|
@ -666,10 +666,18 @@ Involved scientist''')
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def index(self):
|
def index(self):
|
||||||
|
if self:
|
||||||
self.field_indexer = " ".join([
|
self.field_indexer = " ".join([
|
||||||
self.first_name, self.last_name, self.organisation, self.position, self.biography
|
self.first_name if self and self.first_name else "",
|
||||||
])
|
self.last_name if self and self.last_name else "",
|
||||||
|
self.organisation if self and self.organisation else "",
|
||||||
|
self.position if self and self.position else "",
|
||||||
|
self.biography if self and self.biography else ""
|
||||||
|
] if self else "")
|
||||||
|
self.save()
|
||||||
return True
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
def fullname(self):
|
def fullname(self):
|
||||||
return " ".join([self.title if self.title else '',
|
return " ".join([self.title if self.title else '',
|
||||||
|
|
|
@ -176,11 +176,17 @@ def get_paginated(query_set, page, per_page):
|
||||||
for p in paginator.object_list:
|
for p in paginator.object_list:
|
||||||
filters['country'].append(p.country.short_name)
|
filters['country'].append(p.country.short_name)
|
||||||
for r in p.peoplerange_set.all():
|
for r in p.peoplerange_set.all():
|
||||||
|
if r.range and r.range.range_name:
|
||||||
filters['range'].append(r.range.range_name)
|
filters['range'].append(r.range.range_name)
|
||||||
|
else:
|
||||||
|
print("r.range.range_name is None %s %s" % (r.range_id, r.person_id))
|
||||||
for r in p.fieldspeople_set.all():
|
for r in p.fieldspeople_set.all():
|
||||||
filters['field'].append(r.field.name)
|
filters['field'].append(r.field.name)
|
||||||
for r in p.taxapeople_set.all():
|
for r in p.taxapeople_set.all():
|
||||||
|
if r and r.taxon and r.taxon.name:
|
||||||
filters['taxon'].append(r.taxon.name)
|
filters['taxon'].append(r.taxon.name)
|
||||||
|
else:
|
||||||
|
print("r.taxon.name is None %s %s" % (r.range_id, r.taxon))
|
||||||
filters = {
|
filters = {
|
||||||
'country': sorted(set(filters['country'])),
|
'country': sorted(set(filters['country'])),
|
||||||
'range': sorted(set(filters['range'])),
|
'range': sorted(set(filters['range'])),
|
||||||
|
@ -219,7 +225,7 @@ class SearchView(View):
|
||||||
if len(q_country) > 2:
|
if len(q_country) > 2:
|
||||||
query_set = query_set.filter(country__short_name__icontains=q_country.strip().lower())
|
query_set = query_set.filter(country__short_name__icontains=q_country.strip().lower())
|
||||||
if len(q_range) > 2:
|
if len(q_range) > 2:
|
||||||
ranges_people = PeopleRange.objects.filter(range__name__icontains=q_range.strip().lower())
|
ranges_people = PeopleRange.objects.filter(range__range_name__icontains=q_range.strip().lower())
|
||||||
r_people_ids = [rp.person_id for rp in ranges_people]
|
r_people_ids = [rp.person_id for rp in ranges_people]
|
||||||
query_set = query_set.filter(id__in=r_people_ids)
|
query_set = query_set.filter(id__in=r_people_ids)
|
||||||
if len(q_field) > 2:
|
if len(q_field) > 2:
|
||||||
|
|
Loading…
Reference in a new issue