From 461ceb9573a0572ab6c65f68c88a727ddeee992e Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 17 Jan 2022 23:54:29 +0530 Subject: [PATCH 01/64] Organization: Reorganize fields --- app/admin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/admin.py b/app/admin.py index 4681234..760e64d 100644 --- a/app/admin.py +++ b/app/admin.py @@ -211,7 +211,8 @@ class OrganizationAdmin(admin.ModelAdmin): 'fields': ( 'org_num1', ('acronym', 'category', 'subject'), - 'organisation_english', + 'country', + ('organisation_english','tel', 'email', 'url',), ('organisation_2', 'organisation_3'), 'organisation_original', ) @@ -220,13 +221,12 @@ class OrganizationAdmin(admin.ModelAdmin): 'classes': ('collapse',), 'fields': ( ('street', 'postcode', 'city'), - ('po_box', 'country', 'lat_long'), + ('po_box', 'lat_long'), ) }), ('Other details', { 'classes': ('collapse',), - 'fields': (('tel', 'email'), - 'url', + 'fields': ( 'tags', ), }) From 6715924065b04c2116c2125ebbcc2ea533d48215 Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 17 Jan 2022 23:56:48 +0530 Subject: [PATCH 02/64] Add migrations --- ...alter_person_mr_mrs_alter_resource_type.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 app/migrations/0057_alter_person_mr_mrs_alter_resource_type.py diff --git a/app/migrations/0057_alter_person_mr_mrs_alter_resource_type.py b/app/migrations/0057_alter_person_mr_mrs_alter_resource_type.py new file mode 100644 index 0000000..470b763 --- /dev/null +++ b/app/migrations/0057_alter_person_mr_mrs_alter_resource_type.py @@ -0,0 +1,23 @@ +# Generated by Django 4.0.1 on 2022-01-17 18:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0056_alter_mountainrange_range_alternate_id'), + ] + + operations = [ + migrations.AlterField( + model_name='person', + name='mr_mrs', + field=models.TextField(blank=True, choices=[('', ''), ('Ms', 'Ms'), ('Mr', 'Mr')], null=True), + ), + migrations.AlterField( + model_name='resource', + name='type', + field=models.TextField(blank=True, choices=[('', ''), ('Atlas', 'Atlas'), ('Book', 'Book'), ('Book chapter', 'Book chapter'), ('Case study', 'Case study'), ('Conservation project', 'Conservation project'), ('Dataset', 'Dataset'), ('Development project', 'Development project'), ('Journal article', 'Journal article'), ('LTER', 'LTER'), ('Map', 'Map'), ('Network', 'Network'), ('PEGASuS', 'PEGASuS'), ('Presentation', 'Presentation'), ('Report', 'Report'), ('Research project', 'Research project'), ('Research Site', 'Research Site'), ('Thesis', 'Thesis'), ('Video', 'Video'), ('Website', 'Website'), ('Working Group', 'Working Group')], null=True), + ), + ] From e95c87e99c3bfd872df62d9f380f1eb7f3f108a8 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 19 Jan 2022 12:28:22 +0530 Subject: [PATCH 03/64] Organization: revert back organization url --- app/admin.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/app/admin.py b/app/admin.py index 911d276..471efd5 100644 --- a/app/admin.py +++ b/app/admin.py @@ -22,7 +22,7 @@ class PersonAdmin(admin.ModelAdmin): PeopleResourceInline, PeopleRangeInline ] - readonly_fields = ['id', 'organization_url'] + readonly_fields = ['id'] search_fields = ['title', 'first_name', 'last_name', 'organisation', 'position', 'country__short_name', 'contact_email'] fieldsets = ( (None, { @@ -31,7 +31,7 @@ class PersonAdmin(admin.ModelAdmin): ('mr_mrs', 'title','full_name', 'position'), ('first_name', 'last_name', 'search_name', 'status'), ('contact_email', 'email_2', 'skype', 'professional_phone'), - ('organization', 'organization_url'), + 'organization', 'personal_url', 'biography', 'field_of_expertise', @@ -69,11 +69,6 @@ class PersonAdmin(admin.ModelAdmin): ordering = ['full_name'] # list_display_links = ['id'] - def organization_url(self, instance): - return format_html('{1}', - str(instance.organization.url).strip("#") if instance.organization.url else "", - str(instance.organization.url).strip("#") if instance.organization.url else "") - class ResourceKeywordInline(admin.TabularInline): autocomplete_fields = ['keyword'] From edb860fa1673a2f9f7f59bd02afb275e5c3f7476 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 19 Jan 2022 12:34:28 +0530 Subject: [PATCH 04/64] Update Changelog for 1.12 --- Changelog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Changelog.md b/Changelog.md index 803958e..f099f2b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,13 @@ # CHANGELOG.md +## 1.12 (2022-01-19) + +Bugfixes: + - issue#10112: Mountain Range admin form reorganization + - issue#10149: Fix for organization name empty case + - issue#10147: Fix atlas showing up as default type for many Resources + - issue#10142: Person admin form enhancements + ## 1.11 (2022-01-05) Bugfixes: From 41a2129e9af16f692f2de2b5d1a370a0f70e2884 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 00:43:44 +0530 Subject: [PATCH 05/64] Improve refresh_data --- app/convert.py | 74 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/app/convert.py b/app/convert.py index 22ea790..bf61193 100644 --- a/app/convert.py +++ b/app/convert.py @@ -39,7 +39,7 @@ def add_linked(person, obj, data): tgt.name = n tgt.save() field.append(tgt) - return field + return field # Fetch an object by source_id (numeric identifier used in source DB) @@ -71,7 +71,8 @@ def reindex_data(): # Data update routine -def refresh_data(filename, fmt=None): +def refresh_data(filename, fmt=None, update_existing=False): + print("refresh_data") count = 0 rowcount = 0 if not isfile(filename): @@ -101,6 +102,7 @@ def refresh_data(filename, fmt=None): return None if fmt['dataformat'] is DataFormat.PERSON_DETAIL: + print(row) person, source_id = get_by_id(row['ID'], Person) if not person: person = Person.objects.filter(first_name=row['First name'], last_name=row['Last name']).first() @@ -108,28 +110,37 @@ def refresh_data(filename, fmt=None): person = Person(first_name=row['First name'], last_name=row['Last name'], source_id=row['ID']) # Update data fields - person.source_id = source_id - person.title = row['Title'] - person.organisation = row['Organisation English'] - print("Country = %s" % row['country']) - if row['country'] is None or row['country'].strip() == '': - row['country'] = 0 - c = Country.objects.get(id=row['country']) - person.country = c - person.position = row['Position'] - person.biography = row['Biography'] - person.contact_email = row['e-mail 1'] - person.personal_url = fix_url(row['URL']) - + if update_existing: + person.source_id = source_id + person.title = row['Title'] + print(row) + person.organisation = row['Organisation English'] + print("Country = %s" % row['country'] if 'country' in row else '') + if 'country' not in row or row['country'] is None or row['country'].strip() == '': + row['country'] = 0 + c = Country.objects.get(id=row['country']) + person.country = c + person.position = row['Position'] + person.biography = row['Biography'] + person.contact_email = row['e-mail 1'] + person.personal_url = fix_url(row['URL']) + person.save() with transaction.atomic(): research_methods = add_linked(person, Method, row['Methods']) - methods_people = [MethodsPeople.objects.get_or_create(method_id=m.id, person_id=person.id) for m in research_methods] + methods_people = [MethodsPeople.objects.get_or_create(method_id=m.id, person_id=person.id)[0] for m in research_methods] + person.methodspeople_set.set(methods_people) research_scales = add_linked(person, Scale, row['Scale']) - scales_people = [ScalesPeople.objects.get_or_create(scale_id=s.id, person_id=person.id) for s in research_scales] + scales_people = [ScalesPeople.objects.get_or_create(scale_id=s.id, person_id=person.id)[0] for s in research_scales] + person.scalespeople_set.set(scales_people) research_taxa = add_linked(person, Taxon, row['Taxa']) - taxa_people = [TaxaPeople.objects.get_or_create(taxon_id=t.id, person_id=person.id) for t in research_taxa] + taxa_people = [TaxaPeople.objects.get_or_create(taxon_id=t.id, person_id=person.id)[0] for t in research_taxa] + person.taxapeople_set.set(taxa_people) research_fields = add_linked(person, Field, row['Field of expertise']) - fields_people = [FieldsPeople.objects.get_or_create(field_id=f.id, person_id=person.id) for f in research_fields] + fields_people = [FieldsPeople.objects.get_or_create(field_id=f.id, person_id=person.id)[0] for f in research_fields] + person.fieldspeople_set.set(fields_people) + # research_ranges = add_linked(person, MountainRange, row['MountainRange']) + # ranges_people = [RangesPeople.objects.get_or_create(range_id=r.id, person_id=person.id)[0] for r in research_ranges] + # person.rangespeople_set.set(ranges_people) person.index() person.save() count = count + 1 @@ -167,15 +178,30 @@ def refresh_data(filename, fmt=None): elif fmt['dataformat'] is DataFormat.PERSON_RANGE: rzs, source_id = get_by_id(row['MountainRange'], MountainRange, first=False) + print(" range=%s, source_id=%s" % (rzs, source_id)) if not rzs or not rzs.first(): + print(" --- No rzs, continue") continue ppl, source_id = get_by_id(row['Person'], Person, first=False) - if not ppl or not ppl.first(): continue - for person in ppl: - person.ranges = [] - for r in rzs: - person.ranges.append(r) + print(" +++ ppl=%s, source_id=%s" % (ppl, source_id)) + if not ppl or not ppl.first(): + print(" --- No ppl, continue") + continue + with transaction.atomic(): + person = ppl.first() + research_ranges = add_linked(person, MountainRange, row['MountainRange']) + ranges_people = [PeopleRange.objects.get_or_create(range_id=r.id, person_id=person.id)[0] for r in research_ranges] + person.peoplerange_set.set(ranges_people) + # for person in ppl: + # research_ranges = add_linked(person, MountainRange, row['MountainRange']) + # ranges_people = [RangesPeople.objects.get_or_create(range_id=r.id, person_id=person.id)[0] for r in research_ranges] + # person.rangespeople_set.set(ranges_people) + # ranges_people = RangesPeople + # for r in rzs: + # ranges_people.append(r) + # person.rangespeople_set.set(ranges_people) person.save() + #print(" *** Saved %s => %s (%s)" % (person, ranges_people, len(ranges_people))) count = count + 1 elif fmt['extension'] == 'geojson': From a0c3dd57dd923f7126669173c1eadb66aec17331 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 00:44:07 +0530 Subject: [PATCH 06/64] Format + improve indexer --- app/convert.py | 9 ++++++--- app/models.py | 15 +++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/app/convert.py b/app/convert.py index bf61193..faba6e9 100644 --- a/app/convert.py +++ b/app/convert.py @@ -147,7 +147,8 @@ def refresh_data(filename, fmt=None, update_existing=False): elif fmt['dataformat'] is DataFormat.RESOURCE_DETAIL: res, source_id = get_by_id(row['ID'], Resource) - if not res: res = Resource(source_id=source_id) + if not res: + res = Resource(source_id=source_id) res.title = row['Title'] res.citation = row['Citation'] res.url = fix_url(row['URL'].strip('#')) # remove weird #formatting# @@ -166,9 +167,11 @@ def refresh_data(filename, fmt=None, update_existing=False): elif fmt['dataformat'] is DataFormat.PERSON_RESOURCE: rzs, source_id = get_by_id(row['Resource'], Resource, first=False) - if not rzs or not rzs.first(): continue + if not rzs or not rzs.first(): + continue ppl, source_id = get_by_id(row['Person'], Person, first=False) - if not ppl or not ppl.first(): continue + if not ppl or not ppl.first(): + continue for person in ppl: person.resources = [] for r in rzs: diff --git a/app/models.py b/app/models.py index 665e953..bc71105 100644 --- a/app/models.py +++ b/app/models.py @@ -666,10 +666,17 @@ Involved scientist''') return False def index(self): - self.field_indexer = " ".join([ - self.first_name, self.last_name, self.organisation, self.position, self.biography - ]) - return True + if self: + self.field_indexer = " ".join([ + self.first_name if self and self.first_name else "", + self.last_name if self and self.last_name else "", + self.organisation if self and self.organisation else "", + self.position if self and self.position else "", + self.biography if self and self.biography else "" + ] if self else []) + return True + else: + return False def fullname(self): return " ".join([self.title if self.title else '', From 27ecafa88727ce6d88adcc998fa628c801269ae4 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 01:01:28 +0530 Subject: [PATCH 07/64] Debug --- app/convert.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/convert.py b/app/convert.py index faba6e9..5f0b48d 100644 --- a/app/convert.py +++ b/app/convert.py @@ -106,8 +106,10 @@ def refresh_data(filename, fmt=None, update_existing=False): person, source_id = get_by_id(row['ID'], Person) if not person: person = Person.objects.filter(first_name=row['First name'], last_name=row['Last name']).first() + print("Fetched from DB") if not person: person = Person(first_name=row['First name'], last_name=row['Last name'], source_id=row['ID']) + print("Created") # Update data fields if update_existing: From 401b554a1c758824318d659552a8e5e620ad0f34 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 01:05:23 +0530 Subject: [PATCH 08/64] More debug + create person and update fields if it does not exist in DB --- app/convert.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/app/convert.py b/app/convert.py index 5f0b48d..95a070d 100644 --- a/app/convert.py +++ b/app/convert.py @@ -106,9 +106,26 @@ def refresh_data(filename, fmt=None, update_existing=False): person, source_id = get_by_id(row['ID'], Person) if not person: person = Person.objects.filter(first_name=row['First name'], last_name=row['Last name']).first() - print("Fetched from DB") + if person: + print("Fetched from DB") + else: + print("Does not exist in DB") if not person: person = Person(first_name=row['First name'], last_name=row['Last name'], source_id=row['ID']) + person.source_id = source_id + person.title = row['Title'] + print(row) + person.organisation = row['Organisation English'] + print("Country = %s" % row['country'] if 'country' in row else '') + if 'country' not in row or row['country'] is None or row['country'].strip() == '': + row['country'] = 0 + c = Country.objects.get(id=row['country']) + person.country = c + person.position = row['Position'] + person.biography = row['Biography'] + person.contact_email = row['e-mail 1'] + person.personal_url = fix_url(row['URL']) + person.save() print("Created") # Update data fields From 2dda215ffe58e1936881e276413fed54f1886bab Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 01:23:08 +0530 Subject: [PATCH 09/64] Append range name only if not none --- app/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/views.py b/app/views.py index f896b0d..37ec12c 100644 --- a/app/views.py +++ b/app/views.py @@ -176,7 +176,10 @@ def get_paginated(query_set, page, per_page): for p in paginator.object_list: filters['country'].append(p.country.short_name) for r in p.peoplerange_set.all(): - filters['range'].append(r.range.range_name) + if r.range and r.range.range_name: + filters['range'].append(r.range.range_name) + else: + print("r.range.range_name is None %s %s" % (r.range_id, r.person_id)) for r in p.fieldspeople_set.all(): filters['field'].append(r.field.name) for r in p.taxapeople_set.all(): From da8fba3497249c39eb3401b78491b6858449e65a Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 01:28:29 +0530 Subject: [PATCH 10/64] Fix taxon --- app/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/views.py b/app/views.py index 37ec12c..b1877b1 100644 --- a/app/views.py +++ b/app/views.py @@ -183,7 +183,10 @@ def get_paginated(query_set, page, per_page): for r in p.fieldspeople_set.all(): filters['field'].append(r.field.name) for r in p.taxapeople_set.all(): - filters['taxon'].append(r.taxon.name) + if r.taxon and r.taxon.name: + filters['taxon'].append(r.taxon.name) + else: + print("r.taxon.name is None %s %s" % (r.range_id, r.taxon)) filters = { 'country': sorted(set(filters['country'])), 'range': sorted(set(filters['range'])), From 5b7e4eed3e6d05209dc2dce522695b204ef4967c Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 01:32:47 +0530 Subject: [PATCH 11/64] Fix possible none --- app/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views.py b/app/views.py index b1877b1..6373f20 100644 --- a/app/views.py +++ b/app/views.py @@ -183,7 +183,7 @@ def get_paginated(query_set, page, per_page): for r in p.fieldspeople_set.all(): filters['field'].append(r.field.name) for r in p.taxapeople_set.all(): - if r.taxon and r.taxon.name: + if r and r.taxon and r.taxon.name: filters['taxon'].append(r.taxon.name) else: print("r.taxon.name is None %s %s" % (r.range_id, r.taxon)) From f272481bf63790382aff539716ce1d1d8ec5c1e3 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 5 Mar 2022 02:09:38 +0530 Subject: [PATCH 12/64] Fix finding range from name --- app/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views.py b/app/views.py index 6373f20..5819063 100644 --- a/app/views.py +++ b/app/views.py @@ -225,7 +225,7 @@ class SearchView(View): if len(q_country) > 2: query_set = query_set.filter(country__short_name__icontains=q_country.strip().lower()) if len(q_range) > 2: - ranges_people = PeopleRange.objects.filter(range__name__icontains=q_range.strip().lower()) + ranges_people = PeopleRange.objects.filter(range__range_name__icontains=q_range.strip().lower()) r_people_ids = [rp.person_id for rp in ranges_people] query_set = query_set.filter(id__in=r_people_ids) if len(q_field) > 2: From 2c83c3e322ad8e1e36f1f8ec91685f8f8c503cd4 Mon Sep 17 00:00:00 2001 From: PCoder Date: Fri, 11 Mar 2022 16:53:15 +0530 Subject: [PATCH 13/64] Add save method to index --- app/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index bc71105..bed7292 100644 --- a/app/models.py +++ b/app/models.py @@ -673,7 +673,8 @@ Involved scientist''') self.organisation if self and self.organisation else "", self.position if self and self.position else "", self.biography if self and self.biography else "" - ] if self else []) + ] if self else "") + self.save() return True else: return False From 202b6c009201e4d2cee5fe8d32fb9d467fa81c68 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 17 Mar 2022 14:03:35 +0530 Subject: [PATCH 14/64] Some pending fixes --- app/management/commands/import.py | 2 +- app/views.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/app/management/commands/import.py b/app/management/commands/import.py index e7d83cb..b9eea00 100644 --- a/app/management/commands/import.py +++ b/app/management/commands/import.py @@ -356,7 +356,7 @@ class Command(BaseCommand): for k, v in self.csv_files_models_dict.items(): if v.strip().lower() == model_name.strip().lower(): csv_file_name = k - if k == '': + if csv_file_name == '': raise Exception('Could not find a csv file name for model %s' % model_name) if csv_folder_path.endswith('/'): file_path = '%s%s' % (csv_folder_path, csv_file_name) diff --git a/app/views.py b/app/views.py index f896b0d..7198848 100644 --- a/app/views.py +++ b/app/views.py @@ -176,11 +176,17 @@ def get_paginated(query_set, page, per_page): for p in paginator.object_list: filters['country'].append(p.country.short_name) for r in p.peoplerange_set.all(): - filters['range'].append(r.range.range_name) + if r.range and r.range.range_name: + filters['range'].append(r.range.range_name) + else: + print("r.range.range_name is None %s %s" % (r.range_id, r.person_id)) for r in p.fieldspeople_set.all(): filters['field'].append(r.field.name) for r in p.taxapeople_set.all(): - filters['taxon'].append(r.taxon.name) + if r and r.taxon and r.taxon.name: + filters['taxon'].append(r.taxon.name) + else: + print("r.taxon.name is None %s %s" % (r.range_id, r.taxon)) filters = { 'country': sorted(set(filters['country'])), 'range': sorted(set(filters['range'])), @@ -219,7 +225,7 @@ class SearchView(View): if len(q_country) > 2: query_set = query_set.filter(country__short_name__icontains=q_country.strip().lower()) if len(q_range) > 2: - ranges_people = PeopleRange.objects.filter(range__name__icontains=q_range.strip().lower()) + ranges_people = PeopleRange.objects.filter(range__range_name__icontains=q_range.strip().lower()) r_people_ids = [rp.person_id for rp in ranges_people] query_set = query_set.filter(id__in=r_people_ids) if len(q_field) > 2: @@ -239,8 +245,8 @@ class PeopleDetailView(View): person = get_object_or_404(Person, id=people_id) return_data = { 'data': person.dict(), - 'resources': [r.resource.dict() for r in person.resourcespeople_set.all()], - 'ranges': [r.range.dict() for r in person.rangespeople_set.all()], + 'resources': [r.resource.dict() for r in person.peopleresource_set.all()], + 'ranges': [r.range.dict() for r in person.peoplerange_set.all()], 'fields': [r.field.name for r in person.fieldspeople_set.all()], 'methods': [r.method.name for r in person.methodspeople_set.all()], 'scales': [r.scale.name for r in person.scalespeople_set.all()], From 7efa5b3741ffa38a49846ba0cbe020e4b0b229af Mon Sep 17 00:00:00 2001 From: Sami Date: Thu, 17 Mar 2022 16:27:42 +0500 Subject: [PATCH 15/64] formspree emails sent to test.gamba.ungleich@gmail.com and form hides --- app/static/app/gmba-search.tag | 13 +++++++++++-- app/templates/app/admin/base.html | 2 ++ app/templates/app/index.html | 2 ++ app/templates/app/offline.html | 2 ++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/static/app/gmba-search.tag b/app/static/app/gmba-search.tag index c8cc28c..2fc253a 100644 --- a/app/static/app/gmba-search.tag +++ b/app/static/app/gmba-search.tag @@ -248,7 +248,7 @@

Contact

-
@@ -256,7 +256,7 @@

- +

@@ -287,6 +287,12 @@ const FILTER_BLANK = { 'country': [], 'range': [], 'field': [], 'taxon': [] } + + FormSubmitted(){ + console.log("i got executed") + var self = this + self.closedetails() + } this.filters_shown = FILTER_BLANK search(e, nextpage) { @@ -481,6 +487,9 @@ } }) + + + diff --git a/app/templates/app/admin/base.html b/app/templates/app/admin/base.html index c1e0db0..9d6100a 100644 --- a/app/templates/app/admin/base.html +++ b/app/templates/app/admin/base.html @@ -9,6 +9,8 @@ + + diff --git a/app/templates/app/index.html b/app/templates/app/index.html index 4f468a8..6b279ee 100644 --- a/app/templates/app/index.html +++ b/app/templates/app/index.html @@ -6,6 +6,8 @@ + + diff --git a/app/templates/app/offline.html b/app/templates/app/offline.html index 5faa717..775d3c8 100644 --- a/app/templates/app/offline.html +++ b/app/templates/app/offline.html @@ -5,6 +5,8 @@ + + From dffd00d8cb8095b926f92ebc2028711c659335f8 Mon Sep 17 00:00:00 2001 From: Sami Date: Tue, 22 Mar 2022 23:05:59 +0500 Subject: [PATCH 16/64] form submission to client at https://formspree.io/f/gmba@ips.unibe.ch --- app/static/app/gmba-search.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/static/app/gmba-search.tag b/app/static/app/gmba-search.tag index 2fc253a..855cc9a 100644 --- a/app/static/app/gmba-search.tag +++ b/app/static/app/gmba-search.tag @@ -248,7 +248,7 @@

Contact

-
From cc66cfd4d89f86749f28f6bde3c818ad1f815b5d Mon Sep 17 00:00:00 2001 From: Sami Date: Tue, 22 Mar 2022 23:16:20 +0500 Subject: [PATCH 17/64] newform submission to client at https://formspree.io/f/xvolqjpe --- app/static/app/gmba-search.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/static/app/gmba-search.tag b/app/static/app/gmba-search.tag index 855cc9a..c4136c7 100644 --- a/app/static/app/gmba-search.tag +++ b/app/static/app/gmba-search.tag @@ -248,7 +248,7 @@

Contact

- From 0136386ad0237bb4270ea30238521fb2a5e361a0 Mon Sep 17 00:00:00 2001 From: Sami Date: Fri, 25 Mar 2022 23:08:02 +0500 Subject: [PATCH 18/64] Enable CSV export for the 4 main tables Organisations, Persons, Resources, Mountain ranges --- app/admin.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/app/admin.py b/app/admin.py index 471efd5..e9736cc 100644 --- a/app/admin.py +++ b/app/admin.py @@ -1,10 +1,28 @@ +from re import A +from unicodedata import name from django.contrib import admin from django.utils.html import format_html from .models import * from django.forms import TextInput from django.conf import settings +from django.http import HttpResponse +import csv +def download_csv(modeladmin, request, queryset): + name = queryset[0].__class__.__name__ + response = HttpResponse() + response['Content-Disposition'] = "attachment; filename="+name+"_table.csv" + first_row = [] + for i in queryset[0]._meta.__dict__.get("fields"): + temp = str(i).split('.') + first_row.append(temp[-1]) + writer = csv.writer(response) + writer.writerow(first_row) + for dic in queryset.values(): + temp_list = (list(dic.values())) + writer.writerow(temp_list) + return response class PeopleResourceInline(admin.TabularInline): autocomplete_fields = ['resource'] model = PeopleResource @@ -67,6 +85,8 @@ class PersonAdmin(admin.ModelAdmin): autocomplete_fields = ['organization'] list_per_page = settings.ADMIN_LIST_PER_PAGE ordering = ['full_name'] + actions = [download_csv] + # list_display_links = ['id'] @@ -91,6 +111,10 @@ class ResourcePeopleInline(admin.TabularInline): } + + + + class ResourceAdmin(admin.ModelAdmin): inlines = [ ResourceKeywordInline, @@ -130,6 +154,8 @@ class ResourceAdmin(admin.ModelAdmin): }), ) list_per_page = settings.ADMIN_LIST_PER_PAGE + actions = [download_csv] + class RangeNameTranslationInline(admin.TabularInline): @@ -185,6 +211,7 @@ class RangeAdmin(admin.ModelAdmin): ordering = ['range_name'] list_per_page = settings.ADMIN_LIST_PER_PAGE + actions = [download_csv] class PeopleOrganizationInline(admin.TabularInline): model = Person @@ -225,6 +252,7 @@ class OrganizationAdmin(admin.ModelAdmin): }) ) ordering = ['organisation_english'] + actions = [download_csv] def org_url(self, instance): return format_html('{1}', From e9ec1909b4b49d9fd18acae80a754eec70804070 Mon Sep 17 00:00:00 2001 From: Sami Date: Fri, 25 Mar 2022 23:35:38 +0500 Subject: [PATCH 19/64] download_csv name changed to export (.csv) --- app/admin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/admin.py b/app/admin.py index e9736cc..123879a 100644 --- a/app/admin.py +++ b/app/admin.py @@ -23,6 +23,9 @@ def download_csv(modeladmin, request, queryset): temp_list = (list(dic.values())) writer.writerow(temp_list) return response + +download_csv.short_description = 'export (.csv)' + class PeopleResourceInline(admin.TabularInline): autocomplete_fields = ['resource'] model = PeopleResource From d436650d2c4582fe42dbf35c35a3745d59649e54 Mon Sep 17 00:00:00 2001 From: Sami Date: Sun, 27 Mar 2022 17:44:32 +0500 Subject: [PATCH 20/64] [Persons] Offer caculated autocomplete identifiers 10146 --- app/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index bed7292..052578d 100644 --- a/app/models.py +++ b/app/models.py @@ -426,10 +426,15 @@ class Organisation(models.Model): if self.organisation_english: if self.organisation_2: name = f"{self.organisation_english} ({self.organisation_2})" + name = f"{self.country}-{self.organisation_english}-{self.organisation_2}-{self.organisation_3}-{self.acronym}" + else: name = f"{self.organisation_english}" + name = f"{self.country}-{self.organisation_english}-{self.organisation_2}-{self.organisation_3}-{self.acronym}" + + else: - name = '---' + name = ' ---' return name From e8f8ee8ad3e0f8cbbd93f63309e5c6922e0d9db3 Mon Sep 17 00:00:00 2001 From: Sami Date: Sun, 27 Mar 2022 23:32:26 +0500 Subject: [PATCH 21/64] form to override autocomplete organization field for Person class in admin --- app/admin.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/app/admin.py b/app/admin.py index 123879a..b80c041 100644 --- a/app/admin.py +++ b/app/admin.py @@ -7,6 +7,21 @@ from django.forms import TextInput from django.conf import settings from django.http import HttpResponse import csv +from django import forms +from django.contrib.admin.widgets import AutocompleteSelect + +class autocomplete_fields_form(forms.ModelForm): + """This form overrides autocomplete + organization field for Person class + """ + class Meta: + widgets = { + 'my_field': AutocompleteSelect( + Person._meta.get_field('organization').remote_field, + admin.site, + attrs={'data-dropdown-auto-width': 'true'} + ), + } def download_csv(modeladmin, request, queryset): name = queryset[0].__class__.__name__ @@ -85,13 +100,13 @@ class PersonAdmin(admin.ModelAdmin): }), ) list_display = ['id', 'full_name', 'organization'] - autocomplete_fields = ['organization'] + #autocomplete_fields = ['organization'] + form = autocomplete_fields_form # adjust width of autocomplete_fields organization + list_per_page = settings.ADMIN_LIST_PER_PAGE ordering = ['full_name'] actions = [download_csv] -# list_display_links = ['id'] - class ResourceKeywordInline(admin.TabularInline): autocomplete_fields = ['keyword'] From 9dc9daa1e089fdd14d97f6ea61689cade73def3e Mon Sep 17 00:00:00 2001 From: Sami Date: Mon, 28 Mar 2022 10:46:30 +0500 Subject: [PATCH 22/64] form to override autocomplete organization changed --- app/admin.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/app/admin.py b/app/admin.py index b80c041..2be7202 100644 --- a/app/admin.py +++ b/app/admin.py @@ -9,19 +9,24 @@ from django.http import HttpResponse import csv from django import forms from django.contrib.admin.widgets import AutocompleteSelect - -class autocomplete_fields_form(forms.ModelForm): +class autocomplete_fields_form(forms.ModelForm): """This form overrides autocomplete organization field for Person class """ + def __init__(self, *args, **kwargs): + super(autocomplete_fields_form, self).__init__(*args, **kwargs) + + # self.fields['organization'].label = 'My new label' + + class Meta: - widgets = { - 'my_field': AutocompleteSelect( - Person._meta.get_field('organization').remote_field, - admin.site, - attrs={'data-dropdown-auto-width': 'true'} - ), - } + model = Person + fields = '__all__' + widgets={ + 'organization': forms.Select(attrs={'style': 'width:750px'}) + } + + def download_csv(modeladmin, request, queryset): name = queryset[0].__class__.__name__ From 808f1d94bb4ac2db302f53e9097dec219facce10 Mon Sep 17 00:00:00 2001 From: Sami Date: Tue, 12 Apr 2022 12:01:31 +0500 Subject: [PATCH 23/64] alphabatecial order for query in person admin addition for country and dept --- app/admin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/admin.py b/app/admin.py index 2be7202..4ee8e09 100644 --- a/app/admin.py +++ b/app/admin.py @@ -105,6 +105,7 @@ class PersonAdmin(admin.ModelAdmin): }), ) list_display = ['id', 'full_name', 'organization'] + #autocomplete_fields = ['organization'] form = autocomplete_fields_form # adjust width of autocomplete_fields organization @@ -112,7 +113,11 @@ class PersonAdmin(admin.ModelAdmin): ordering = ['full_name'] actions = [download_csv] - + def get_field_queryset(self, db, db_field, request): + queryset = super().get_field_queryset(db, db_field, request) + if db_field.name == 'organization': + queryset = queryset.order_by('country','organisation_english') + return queryset class ResourceKeywordInline(admin.TabularInline): autocomplete_fields = ['keyword'] model = ResourceKeyword From 8e409d1a384f5e8aac3ff25e8e8cd4a1cad97254 Mon Sep 17 00:00:00 2001 From: Sami Date: Tue, 12 Apr 2022 12:47:17 +0500 Subject: [PATCH 24/64] space changed too < in __str__ person --- app/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models.py b/app/models.py index 052578d..22f7e7d 100644 --- a/app/models.py +++ b/app/models.py @@ -426,11 +426,11 @@ class Organisation(models.Model): if self.organisation_english: if self.organisation_2: name = f"{self.organisation_english} ({self.organisation_2})" - name = f"{self.country}-{self.organisation_english}-{self.organisation_2}-{self.organisation_3}-{self.acronym}" + name = f"{self.country} < {self.organisation_english} < {self.organisation_2} < {self.organisation_3} < {self.acronym}" else: name = f"{self.organisation_english}" - name = f"{self.country}-{self.organisation_english}-{self.organisation_2}-{self.organisation_3}-{self.acronym}" + name = f"{self.country} < {self.organisation_english} < {self.organisation_2} < {self.organisation_3} < {self.acronym}" else: From c84ae636c5f562773d1b73849bff842b6463658a Mon Sep 17 00:00:00 2001 From: Sami Date: Tue, 12 Apr 2022 13:31:14 +0500 Subject: [PATCH 25/64] added organisation and acronyms in admin query field --- app/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/admin.py b/app/admin.py index 4ee8e09..8fecf28 100644 --- a/app/admin.py +++ b/app/admin.py @@ -116,7 +116,7 @@ class PersonAdmin(admin.ModelAdmin): def get_field_queryset(self, db, db_field, request): queryset = super().get_field_queryset(db, db_field, request) if db_field.name == 'organization': - queryset = queryset.order_by('country','organisation_english') + queryset = queryset.order_by('country','organisation_english','organisation_2','organisation_3', 'acronym') return queryset class ResourceKeywordInline(admin.TabularInline): autocomplete_fields = ['keyword'] From b7f8e237fa538ec8f08f0699fa09993a73dca159 Mon Sep 17 00:00:00 2001 From: Sami Date: Wed, 18 May 2022 22:12:18 +0500 Subject: [PATCH 26/64] missing search field organisation_english at organizationadmin --- app/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/admin.py b/app/admin.py index 8fecf28..74f184d 100644 --- a/app/admin.py +++ b/app/admin.py @@ -252,7 +252,7 @@ class OrganizationAdmin(admin.ModelAdmin): PeopleOrganizationInline, ] readonly_fields = ('org_num1',) - search_fields = ['organisation_search', 'org_alpha_search', 'organisation_2', 'organisation_3', 'subject'] + search_fields = ['organisation_search', 'org_alpha_search', 'organisation_2', 'organisation_3', 'subject','organisation_english'] list_display = ['org_num1', 'organisation_english', 'organisation_2', 'country'] fieldsets = ( (None, { From c5bfc46e33205dc264fd89ec5fc5d83f1bc6e3b9 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 25 Aug 2022 10:26:17 +0530 Subject: [PATCH 27/64] Add first version of docker files --- Dockerfile | 7 +++++++ docker-compose.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e730dbb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +WORKDIR /code +COPY requirements.txt /code/ +RUN pip install -r requirements.txt +COPY . /code/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c3806f2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.9" + +services: + db: + image: postgres + volumes: + - ./data/db:/var/lib/postgresql/data + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + web: + build: . + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + environment: + - POSTGRES_NAME=app + - POSTGRES_USER=app + - POSTGRES_PASSWORD= + depends_on: + - db + + From 7fd30a5dcd752995d00ecfe17ee0203ebdf224c4 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 25 Aug 2022 11:03:25 +0530 Subject: [PATCH 28/64] Add production ready Dockerfle and entrypoint files --- Dockerfile.prod | 71 ++++++++++++++++++++++++++++++++++++++++++++++ entrypoint.prod.sh | 15 ++++++++++ 2 files changed, 86 insertions(+) create mode 100644 Dockerfile.prod create mode 100755 entrypoint.prod.sh diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 0000000..9b51ad7 --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,71 @@ +########### +# BUILDER # +########### + +# pull official base image +FROM python:3.9.6-alpine as builder + +# set work directory +WORKDIR /usr/src/gmba_django + +# set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# install psycopg2 dependencies +RUN apk update \ + && apk add postgresql-dev gcc python3-dev musl-dev + +# lint +RUN pip install --upgrade pip +RUN pip install flake8==3.9.2 +COPY . . +RUN flake8 --ignore=E501,F401 . + +# install dependencies +COPY ./requirements.txt . +RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/gmba_django/wheels -r requirements.txt + + +######### +# FINAL # +######### + +# pull official base image +FROM python:3.9.6-alpine + +# create directory for the app user +RUN mkdir -p /home/app + +# create the app user +RUN addgroup -S app && adduser -S app -G app + +# create the appropriate directories +ENV HOME=/home/app +ENV APP_HOME=/home/app/app +RUN mkdir $APP_HOME +WORKDIR $APP_HOME + +# install dependencies +RUN apk update && apk add libpq +COPY --from=builder /usr/src/gmba_django/wheels /wheels +COPY --from=builder /usr/src/gmba_django/requirements.txt . +RUN pip install --no-cache /wheels/* + +# copy entrypoint.prod.sh +COPY ./entrypoint.prod.sh . +RUN sed -i 's/\r$//g' $APP_HOME/entrypoint.prod.sh +RUN chmod +x $APP_HOME/entrypoint.prod.sh + +# copy project +COPY . $APP_HOME + +# chown all the files to the app user +RUN chown -R app:app $APP_HOME + +# change to the app user +USER app + +# run entrypoint.prod.sh +ENTRYPOINT ["/home/app/app/entrypoint.prod.sh"] + diff --git a/entrypoint.prod.sh b/entrypoint.prod.sh new file mode 100755 index 0000000..2cc61cd --- /dev/null +++ b/entrypoint.prod.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +if [ "$DATABASE" = "postgres" ] +then + echo "Waiting for postgres..." + + while ! nc -z $SQL_HOST $SQL_PORT; do + sleep 0.1 + done + + echo "PostgreSQL started" +fi + +exec "$@" + From 3b9498d13dc10973ca28a7334eaf59e536e9bfd4 Mon Sep 17 00:00:00 2001 From: app Date: Thu, 25 Aug 2022 08:26:59 +0000 Subject: [PATCH 29/64] WIP: fixing docker compose scritps --- Dockerfile.prod | 2 +- docker-compose.yml | 34 +++++++++++++----------------- gmba_django/settings/production.py | 8 +++---- requirements.txt | 1 + 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 9b51ad7..e389e79 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -20,7 +20,7 @@ RUN apk update \ RUN pip install --upgrade pip RUN pip install flake8==3.9.2 COPY . . -RUN flake8 --ignore=E501,F401 . +#RUN flake8 --ignore=E501,F401 . # install dependencies COPY ./requirements.txt . diff --git a/docker-compose.yml b/docker-compose.yml index c3806f2..96eae14 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,22 @@ -version: "3.9" - +version: '3.8' + services: - db: - image: postgres - volumes: - - ./data/db:/var/lib/postgresql/data - environment: - - POSTGRES_DB=postgres - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres web: - build: . - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - .:/code + build: ./app + command: gunicorn gmba_django.wsgi:application --bind 0.0.0.0:8000 ports: - - "8000:8000" - environment: - - POSTGRES_NAME=app - - POSTGRES_USER=app - - POSTGRES_PASSWORD= + - 8000:8000 + env_file: + - ./.env.prod depends_on: - db + db: + image: postgres:13.0-alpine + volumes: + - postgres_data:/var/lib/postgresql/data/ + env_file: + - ./.env.prod +volumes: + postgres_data: diff --git a/gmba_django/settings/production.py b/gmba_django/settings/production.py index 45a8caf..bcb825e 100644 --- a/gmba_django/settings/production.py +++ b/gmba_django/settings/production.py @@ -8,10 +8,10 @@ DATABASES = { 'default': { 'ENGINE': os.environ.get('SQL_ENGINE', 'django.db.backends.sqlite3'), 'NAME': os.environ.get('SQL_DATABASE', os.path.join(BASE_DIR, 'db.sqlite3')), - # 'USER': os.environ.get('SQL_USER', 'user'), - # 'PASSWORD': os.environ.get('SQL_PASSWORD', 'password'), - # 'HOST': os.environ.get('SQL_HOST', 'localhost'), - # 'PORT': os.environ.get('SQL_PORT', ''), + 'USER': os.environ.get('SQL_USER', 'app'), + 'PASSWORD': os.environ.get('SQL_PASSWORD', ''), + 'HOST': os.environ.get('SQL_HOST', 'localhost'), + 'PORT': os.environ.get('SQL_PORT', ''), } } diff --git a/requirements.txt b/requirements.txt index ad8eda7..ce977aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ sqlparse==0.4.1 typing-extensions==3.10.0.0 psycopg2==2.9.1 Werkzeug==2.0.2 +gunicorn==20.1.0 From f960c4e779daa59b2c7d256a8b9fb3c64f7ed3f4 Mon Sep 17 00:00:00 2001 From: app Date: Thu, 25 Aug 2022 08:27:48 +0000 Subject: [PATCH 30/64] Add docker-compose.prod.yaml and nginx confs --- docker-compose.prod.yml | 34 ++++++++++++++++++++++++++++++++++ nginx/Dockerfile | 4 ++++ nginx/nginx.conf | 15 +++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 docker-compose.prod.yml create mode 100644 nginx/Dockerfile create mode 100644 nginx/nginx.conf diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..8d6c176 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,34 @@ +version: '3.8' + +services: + web: + build: + context: ./ + dockerfile: Dockerfile.prod + command: gunicorn gmba_django.wsgi:application --bind 0.0.0.0:8000 + volumes: + - static_volume:/home/app/app/staticfiles + expose: + - 8000 + env_file: + - ./.env.prod + depends_on: + - db + db: + image: postgres:13.0-alpine + volumes: + - postgres_data:/var/lib/postgresql/data/ + env_file: + - ./.env.prod.db + nginx: + build: ./nginx + volumes: + - static_volume:/home/app/app/staticfiles + ports: + - 1337:80 + depends_on: + - web + +volumes: + postgres_data: + static_volume: diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 0000000..4580e8d --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx:1.21-alpine + +RUN rm /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/conf.d diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..c709070 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,15 @@ +upstream gmba_django { + server 127.0.0.1:8000; +} + +server { + listen 80; + + location / { + proxy_pass http://gmba_django; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + +} From 2c4fb661760853d1557829769ad672e696c06930 Mon Sep 17 00:00:00 2001 From: Sami Date: Wed, 7 Sep 2022 23:48:58 +0500 Subject: [PATCH 31/64] resource and relevant objs del --- app/admin.py | 76 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/app/admin.py b/app/admin.py index 74f184d..80ae861 100644 --- a/app/admin.py +++ b/app/admin.py @@ -9,27 +9,28 @@ from django.http import HttpResponse import csv from django import forms from django.contrib.admin.widgets import AutocompleteSelect + + class autocomplete_fields_form(forms.ModelForm): """This form overrides autocomplete organization field for Person class """ + def __init__(self, *args, **kwargs): super(autocomplete_fields_form, self).__init__(*args, **kwargs) # self.fields['organization'].label = 'My new label' - class Meta: model = Person fields = '__all__' - widgets={ + widgets = { 'organization': forms.Select(attrs={'style': 'width:750px'}) - } - + } def download_csv(modeladmin, request, queryset): - name = queryset[0].__class__.__name__ + name = queryset[0].__class__.__name__ response = HttpResponse() response['Content-Disposition'] = "attachment; filename="+name+"_table.csv" first_row = [] @@ -40,12 +41,14 @@ def download_csv(modeladmin, request, queryset): writer = csv.writer(response) writer.writerow(first_row) for dic in queryset.values(): - temp_list = (list(dic.values())) + temp_list = (list(dic.values())) writer.writerow(temp_list) return response + download_csv.short_description = 'export (.csv)' + class PeopleResourceInline(admin.TabularInline): autocomplete_fields = ['resource'] model = PeopleResource @@ -64,12 +67,13 @@ class PersonAdmin(admin.ModelAdmin): PeopleRangeInline ] readonly_fields = ['id'] - search_fields = ['title', 'first_name', 'last_name', 'organisation', 'position', 'country__short_name', 'contact_email'] + search_fields = ['title', 'first_name', 'last_name', 'organisation', + 'position', 'country__short_name', 'contact_email'] fieldsets = ( (None, { 'fields': ( ('id', 'profile_on_web', 'news_letter'), - ('mr_mrs', 'title','full_name', 'position'), + ('mr_mrs', 'title', 'full_name', 'position'), ('first_name', 'last_name', 'search_name', 'status'), ('contact_email', 'email_2', 'skype', 'professional_phone'), 'organization', @@ -81,7 +85,7 @@ class PersonAdmin(admin.ModelAdmin): ('Research Scale', { 'classes': ('collapse',), 'fields': (('field_site', 'transect', 'mountain_top', 'mountain_range', 'landscape', 'regional', 'national',), - '_global'), + '_global'), }), ('Methods and tools', { 'classes': ('collapse',), @@ -107,7 +111,7 @@ class PersonAdmin(admin.ModelAdmin): list_display = ['id', 'full_name', 'organization'] #autocomplete_fields = ['organization'] - form = autocomplete_fields_form # adjust width of autocomplete_fields organization + form = autocomplete_fields_form # adjust width of autocomplete_fields organization list_per_page = settings.ADMIN_LIST_PER_PAGE ordering = ['full_name'] @@ -116,8 +120,11 @@ class PersonAdmin(admin.ModelAdmin): def get_field_queryset(self, db, db_field, request): queryset = super().get_field_queryset(db, db_field, request) if db_field.name == 'organization': - queryset = queryset.order_by('country','organisation_english','organisation_2','organisation_3', 'acronym') + queryset = queryset.order_by( + 'country', 'organisation_english', 'organisation_2', 'organisation_3', 'acronym') return queryset + + class ResourceKeywordInline(admin.TabularInline): autocomplete_fields = ['keyword'] model = ResourceKeyword @@ -139,10 +146,6 @@ class ResourcePeopleInline(admin.TabularInline): } - - - - class ResourceAdmin(admin.ModelAdmin): inlines = [ ResourceKeywordInline, @@ -182,8 +185,32 @@ class ResourceAdmin(admin.ModelAdmin): }), ) list_per_page = settings.ADMIN_LIST_PER_PAGE - actions = [download_csv] + def get_actions(self, request): + actions = super().get_actions(request) + if 'delete_selected' in actions: + del actions['delete_selected'] + return actions + + def delete_resource(self, request, obj): + for o in obj.all(): + resource_keyword = ResourceKeyword.objects.filter(resource=o.id) + for i in resource_keyword: + i.delete() + resource_range = ResourceRange.objects.filter(resource_title=o.id) + for i in resource_range: + i.delete() + resource_people = ResourcesPeople.objects.filter(resource=o.id) + for i in resource_people: + i.delete() + resource_people_resource = PeopleResource.objects.filter( + resource=o.id) + for i in resource_people_resource: + i.delete() + o.delete() + delete_resource.short_description = 'Delete Resource & Related Fields' + + actions = [download_csv, delete_resource] class RangeNameTranslationInline(admin.TabularInline): @@ -241,6 +268,7 @@ class RangeAdmin(admin.ModelAdmin): actions = [download_csv] + class PeopleOrganizationInline(admin.TabularInline): model = Person exclude = ['source_id', 'organisation', 'field_indexer'] @@ -252,15 +280,17 @@ class OrganizationAdmin(admin.ModelAdmin): PeopleOrganizationInline, ] readonly_fields = ('org_num1',) - search_fields = ['organisation_search', 'org_alpha_search', 'organisation_2', 'organisation_3', 'subject','organisation_english'] - list_display = ['org_num1', 'organisation_english', 'organisation_2', 'country'] + search_fields = ['organisation_search', 'org_alpha_search', + 'organisation_2', 'organisation_3', 'subject', 'organisation_english'] + list_display = ['org_num1', 'organisation_english', + 'organisation_2', 'country'] fieldsets = ( (None, { 'fields': ( 'org_num1', ('acronym', 'category', 'subject'), 'country', - ('organisation_english','tel', 'email', 'url',), + ('organisation_english', 'tel', 'email', 'url',), ('organisation_2', 'organisation_3'), 'organisation_original', ) @@ -275,8 +305,8 @@ class OrganizationAdmin(admin.ModelAdmin): ('Other details', { 'classes': ('collapse',), 'fields': ( - 'tags', - ), + 'tags', + ), }) ) ordering = ['organisation_english'] @@ -373,7 +403,8 @@ class SpeciesAdmin(admin.ModelAdmin): class TaxonRangeAdmin(admin.ModelAdmin): model = TaxonRange - search_fields = ['range__range_name', 'taxon', 'subrange_or_region', 'distribution', 'source', 'remarks'] + search_fields = ['range__range_name', 'taxon', + 'subrange_or_region', 'distribution', 'source', 'remarks'] list_display = ['range_name', 'taxon'] def range_name(self, obj): @@ -447,6 +478,7 @@ admin.site.register(Keyword, KeywordAdmin) # admin.site.register(Scale) # admin.site.register(Taxon) # admin.site.register(Field, FieldAdmin) +admin.site.disable_action('delete_selected') # LU models From 48a25858299ca533b96a85dfcb4330212c954991 Mon Sep 17 00:00:00 2001 From: Sami Date: Thu, 8 Sep 2022 22:44:24 +0500 Subject: [PATCH 32/64] person save method overload --- app/models.py | 160 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 106 insertions(+), 54 deletions(-) diff --git a/app/models.py b/app/models.py index 22f7e7d..8875ee0 100644 --- a/app/models.py +++ b/app/models.py @@ -31,7 +31,8 @@ class Field(models.Model): class FieldsPeople(models.Model): id = models.AutoField(primary_key=True) - person = models.ForeignKey('Person', models.DO_NOTHING, blank=True, null=True) + person = models.ForeignKey( + 'Person', models.DO_NOTHING, blank=True, null=True) field = models.ForeignKey(Field, models.DO_NOTHING, blank=True, null=True) class Meta: @@ -53,8 +54,10 @@ class Method(models.Model): class MethodsPeople(models.Model): id = models.AutoField(primary_key=True) - person = models.ForeignKey('Person', models.DO_NOTHING, blank=True, null=True) - method = models.ForeignKey(Method, models.DO_NOTHING, blank=True, null=True) + person = models.ForeignKey( + 'Person', models.DO_NOTHING, blank=True, null=True) + method = models.ForeignKey( + Method, models.DO_NOTHING, blank=True, null=True) class Meta: db_table = 'methods_people' @@ -92,10 +95,14 @@ class MountainRange(models.Model): #range_name_map = models.CharField(blank=True, null=True, unique=True, max_length=128) range_name = models.CharField(blank=True, null=True, max_length=128) range_name_ascii = models.CharField(blank=True, null=True, max_length=128) - range_name_language = models.ForeignKey(Language, models.DO_NOTHING, blank=True, null=True, to_field='id') - mother_range = models.ForeignKey("self", models.DO_NOTHING, blank=True, null=True, to_field='id') - feature = models.ForeignKey(RangeType, models.DO_NOTHING, blank=True, null=True, to_field='id') - map_unit = models.TextField(blank=True, null=True, choices=MAP_UNIT_CHOICES) + range_name_language = models.ForeignKey( + Language, models.DO_NOTHING, blank=True, null=True, to_field='id') + mother_range = models.ForeignKey( + "self", models.DO_NOTHING, blank=True, null=True, to_field='id') + feature = models.ForeignKey( + RangeType, models.DO_NOTHING, blank=True, null=True, to_field='id') + map_unit = models.TextField( + blank=True, null=True, choices=MAP_UNIT_CHOICES) level = models.TextField(blank=True, null=True) level_text = models.TextField(blank=True, null=True) level_1 = models.TextField(blank=True, null=True) @@ -113,14 +120,16 @@ class MountainRange(models.Model): comments = models.TextField(blank=True, null=True) checked = models.BooleanField(default=False) source = models.TextField(blank=True, null=True) - range_alternate_id = models.CharField(blank=True, null=True, max_length=128) + range_alternate_id = models.CharField( + blank=True, null=True, max_length=128) geologic_region = models.TextField(blank=True, null=True) gmba_v2_id = models.PositiveIntegerField(blank=True, null=True) gmba_v2_id_str = models.TextField(blank=True, null=True) wiki_data_id = models.CharField(blank=True, null=True, max_length=25) wiki_data_url = models.URLField(blank=True, null=True) select_300 = models.BooleanField(default=False) - gmba_narrow = models.BooleanField(default=False, verbose_name="GMBA Standard") + gmba_narrow = models.BooleanField( + default=False, verbose_name="GMBA Standard") name_fr = models.CharField(blank=True, null=True, max_length=128) name_de = models.CharField(blank=True, null=True, max_length=128) name_es = models.CharField(blank=True, null=True, max_length=128) @@ -201,7 +210,8 @@ class Resource(models.Model): PEGASuS_Check_map_with_author = models.BooleanField(default=False) PEGASuS_polygon_ID = models.CharField(max_length=32, blank=True, null=True) PEGASuS_Polygon_comments = models.TextField(blank=True, null=True) - PEGASuS_Assessment_ID = models.CharField(max_length=32, blank=True, null=True) + PEGASuS_Assessment_ID = models.CharField( + max_length=32, blank=True, null=True) gloria = models.BooleanField(default=False) gnomo = models.BooleanField(default=False) lter = models.BooleanField(default=False) @@ -352,15 +362,20 @@ class GMBA_function(models.Model): class Organisation(models.Model): CATEGORY_CHOICES = ( ('Independent', 'Independent'), - ('Public administration > Research Institution', 'Public administration > Research Institution'), + ('Public administration > Research Institution', + 'Public administration > Research Institution'), ('Academia > University > Institute', 'Academia > University > Institute'), ('Academia > University', 'Academia > University'), - ('Public administration > Other Agency', 'Public administration > Other Agency'), + ('Public administration > Other Agency', + 'Public administration > Other Agency'), ('Private > NGO (not for profit)', 'Private > NGO (not for profit)'), ('Academia > Academy of Sciences', 'Academia > Academy of Sciences'), - ('Academia > Academy of Sciences > Institute', 'Academia > Academy of Sciences > Institute'), - ('Public administration > Government', 'Public administration > Government'), - ('Intergovernmental Agency > Research Institution', 'Intergovernmental Agency > Research Institution'), + ('Academia > Academy of Sciences > Institute', + 'Academia > Academy of Sciences > Institute'), + ('Public administration > Government', + 'Public administration > Government'), + ('Intergovernmental Agency > Research Institution', + 'Intergovernmental Agency > Research Institution'), ('Intergovernmental Agency', 'Intergovernmental Agency '), ('Private > Research Institution', 'Private > Research Institution'), ('Collection > Museum', 'Collection > Museum'), @@ -384,7 +399,8 @@ class Organisation(models.Model): ('Culture / arts', 'Culture / arts'), ('Geography', 'Geography'), ('Zoology', 'Zoology'), - ('Development / poverty / human rights', 'Development / poverty / human rights'), + ('Development / poverty / human rights', + 'Development / poverty / human rights'), ('Social Sciences', 'Social Sciences'), ('Earth Sciences', 'Earth Sciences'), ('Health', 'Health'), @@ -399,9 +415,12 @@ class Organisation(models.Model): org_num1 = models.AutoField(primary_key=True, verbose_name='ID') organisation_search = models.TextField(blank=True, null=True) org_alpha_search = models.TextField(blank=True, null=True) - organisation_english = models.CharField(max_length=256, blank=True, null=True, verbose_name='Organisation') - organisation_2 = models.CharField(max_length=256, blank=True, null=True, verbose_name='Department 1') - organisation_3 = models.CharField(max_length=256, blank=True, null=True, verbose_name='Department 2') + organisation_english = models.CharField( + max_length=256, blank=True, null=True, verbose_name='Organisation') + organisation_2 = models.CharField( + max_length=256, blank=True, null=True, verbose_name='Department 1') + organisation_3 = models.CharField( + max_length=256, blank=True, null=True, verbose_name='Department 2') organisation_original = models.TextField(blank=True, null=True) acronym = models.CharField(max_length=128, blank=True, null=True) street = models.CharField(max_length=256, blank=True, null=True) @@ -414,13 +433,16 @@ class Organisation(models.Model): url = models.URLField(blank=True, null=True) tel = models.CharField(max_length=128, blank=True, null=True) email = models.EmailField(blank=True, null=True) - country = models.ForeignKey(Country, models.DO_NOTHING, blank=True, null=True, to_field='id') + country = models.ForeignKey( + Country, models.DO_NOTHING, blank=True, null=True, to_field='id') tags = models.TextField(blank=True, null=True) description = models.TextField(blank=True, null=True) northing = models.TextField(blank=True, null=True) easting = models.TextField(blank=True, null=True) - category = models.TextField(blank=True, null=True, choices=CATEGORY_CHOICES) - subject = models.CharField(max_length=128, blank=True, null=True, choices=SUBJECT_CHOICES) + category = models.TextField( + blank=True, null=True, choices=CATEGORY_CHOICES) + subject = models.CharField( + max_length=128, blank=True, null=True, choices=SUBJECT_CHOICES) def __str__(self): if self.organisation_english: @@ -432,14 +454,12 @@ class Organisation(models.Model): name = f"{self.organisation_english}" name = f"{self.country} < {self.organisation_english} < {self.organisation_2} < {self.organisation_3} < {self.acronym}" - else: name = ' ---' return name - class RangeOnlineInfo(models.Model): id = models.AutoField(primary_key=True) # This is supposed to be linked to MountainRange via range_name_map, but the range_name_map @@ -456,8 +476,10 @@ class RangeOnlineInfo(models.Model): class RangeNameTranslation(models.Model): id = models.AutoField(primary_key=True) - range_name = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) - language_translation = models.ForeignKey(Language, models.DO_NOTHING, blank=True, null=True) + range_name = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) + language_translation = models.ForeignKey( + Language, models.DO_NOTHING, blank=True, null=True) range_name_translation = models.TextField(blank=True, null=True) def __str__(self): @@ -510,8 +532,10 @@ class Keyword(models.Model): class ResourceKeyword(models.Model): id = models.AutoField(primary_key=True) - resource = models.ForeignKey(Resource, models.DO_NOTHING, blank=True, null=True) - keyword = models.ForeignKey(Keyword, models.DO_NOTHING, blank=True, null=True, to_field='keyword_id') + resource = models.ForeignKey( + Resource, models.DO_NOTHING, blank=True, null=True) + keyword = models.ForeignKey( + Keyword, models.DO_NOTHING, blank=True, null=True, to_field='keyword_id') def __str__(self): return '%s - %s' % (self.resource.title, self.keyword.keyword) @@ -534,8 +558,10 @@ class NamesImport(models.Model): class ResourceRange(models.Model): id = models.AutoField(primary_key=True) - resource_title = models.ForeignKey(Resource, models.DO_NOTHING, blank=True, null=True) - range_name = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) + resource_title = models.ForeignKey( + Resource, models.DO_NOTHING, blank=True, null=True) + range_name = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) def __str__(self): return '%s, %s' % (self.resource_title.title, self.range_name.name) @@ -543,7 +569,8 @@ class ResourceRange(models.Model): class LanguageLink(models.Model): id = models.AutoField(primary_key=True) - language_number_code = models.ForeignKey(Language, models.DO_NOTHING, blank=True, null=True) + language_number_code = models.ForeignKey( + Language, models.DO_NOTHING, blank=True, null=True) language_letter_code = models.TextField(blank=True, null=True) def __str__(self): @@ -565,8 +592,10 @@ class Peak(models.Model): class Search(models.Model): id = models.AutoField(primary_key=True) - range_name = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) - repository = models.ForeignKey(Repository, models.DO_NOTHING, blank=True, null=True) + range_name = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) + repository = models.ForeignKey( + Repository, models.DO_NOTHING, blank=True, null=True) search_string = models.TextField(blank=True, null=True) search_date = models.TextField(blank=True, null=True) result = models.TextField(blank=True, null=True) @@ -589,12 +618,12 @@ class Person(models.Model): ('''Involved scientist Steering committee''', '''Involved scientist Steering committee'''), - ('Network', 'Network'), + ('Network', 'Network'), ('Newsletter only', 'Newsletter only'), ('''Steering committee Involved scientist''', '''Steering committee Involved scientist''') - ) + ) id = models.AutoField(primary_key=True) source_id = models.IntegerField(blank=True, null=True) title = models.CharField(blank=True, null=True, max_length=255) @@ -605,7 +634,9 @@ Involved scientist''') contact_email = models.EmailField(blank=True, null=True) # email_1 personal_url = models.URLField(blank=True, null=True) # url ? biography = models.TextField(blank=True, null=True) - field_indexer = models.TextField(db_column='_indexer', blank=True, null=True) # Field renamed because it started with '_'. + # Field renamed because it started with '_'. + field_indexer = models.TextField( + db_column='_indexer', blank=True, null=True) mr_mrs = models.TextField(blank=True, null=True, choices=MR_MRS_CHOICES) full_name = models.CharField(blank=True, null=True, max_length=100) @@ -616,12 +647,16 @@ Involved scientist''') mobile_number = models.CharField(blank=True, null=True, max_length=50) url = models.URLField(blank=True, null=True) field_of_expertise = models.TextField(blank=True, null=True) - status = models.ForeignKey(PeopleStatus, models.DO_NOTHING, blank=True, null=True, to_field='id') + status = models.ForeignKey( + PeopleStatus, models.DO_NOTHING, blank=True, null=True, to_field='id') entry_date = models.TextField(blank=True, null=True) - gmba_function = models.TextField(blank=True, null=True, choices=GMBA_FUNCTION_CHOICES) + gmba_function = models.TextField( + blank=True, null=True, choices=GMBA_FUNCTION_CHOICES) news_letter = models.BooleanField(default=False) - country = models.ForeignKey(Country, models.DO_NOTHING, blank=True, null=True, to_field='id') - organization = models.ForeignKey(Organisation, models.DO_NOTHING, blank=True, null=True) + country = models.ForeignKey( + Country, models.DO_NOTHING, blank=True, null=True, to_field='id') + organization = models.ForeignKey( + Organisation, models.DO_NOTHING, blank=True, null=True) birds = models.BooleanField(default=False) mammals = models.BooleanField(default=False) reptiles = models.BooleanField(default=False) @@ -646,7 +681,8 @@ Involved scientist''') assessment = models.BooleanField(default=False) meta_analysis = models.BooleanField(default=False) synthesis = models.BooleanField(default=False) - qualitative_ssm = models.BooleanField(default=False) #"Qualitative social science methods (interviews, surveys)" + # "Qualitative social science methods (interviews, surveys)" + qualitative_ssm = models.BooleanField(default=False) genetic_analyses = models.BooleanField(default=False) field_site = models.BooleanField(default=False) transect = models.BooleanField(default=False) @@ -679,7 +715,7 @@ Involved scientist''') self.position if self and self.position else "", self.biography if self and self.biography else "" ] if self else "") - self.save() + # self.save() return True else: return False @@ -704,11 +740,16 @@ Involved scientist''') 'biography': self.biography or '', } + def save(self, *args, **kwargs): + self.index() + super(Person, self).save(*args, **kwargs) + class TaxaPeople(models.Model): id = models.AutoField(primary_key=True) person = models.ForeignKey(Person, models.CASCADE, blank=True, null=True) - taxon = models.ForeignKey('Taxon', models.DO_NOTHING, blank=True, null=True) + taxon = models.ForeignKey( + 'Taxon', models.DO_NOTHING, blank=True, null=True) class Meta: db_table = 'taxa_people' @@ -720,7 +761,8 @@ class TaxaPeople(models.Model): class RangesPeople(models.Model): id = models.AutoField(primary_key=True) person = models.ForeignKey(Person, models.CASCADE, blank=True, null=True) - range = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) + range = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) class Meta: db_table = 'ranges_people' @@ -732,7 +774,8 @@ class RangesPeople(models.Model): class ResourcesPeople(models.Model): id = models.AutoField(primary_key=True) person = models.ForeignKey(Person, models.CASCADE, blank=True, null=True) - resource = models.ForeignKey(Resource, models.CASCADE, blank=True, null=True) + resource = models.ForeignKey( + Resource, models.CASCADE, blank=True, null=True) class Meta: db_table = 'resources_people' @@ -743,7 +786,8 @@ class ResourcesPeople(models.Model): class ScalesPeople(models.Model): id = models.AutoField(primary_key=True) - person = models.ForeignKey(Person, models.DO_NOTHING, blank=True, null=True) + person = models.ForeignKey( + Person, models.DO_NOTHING, blank=True, null=True) scale = models.ForeignKey(Scale, models.DO_NOTHING, blank=True, null=True) class Meta: @@ -767,7 +811,8 @@ class Species(models.Model): class PeopleFunction(models.Model): id = models.AutoField(primary_key=True) person = models.ForeignKey(Person, models.CASCADE, blank=True, null=True) - function = models.ForeignKey(GMBA_function, models.DO_NOTHING, blank=True, null=True) + function = models.ForeignKey( + GMBA_function, models.DO_NOTHING, blank=True, null=True) def __str__(self): return self.person.full_name @@ -775,8 +820,10 @@ class PeopleFunction(models.Model): class SpeciesRange(models.Model): id = models.AutoField(primary_key=True) - scientific_name = models.ForeignKey(Species, models.DO_NOTHING, blank=True, null=True) - range = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) + scientific_name = models.ForeignKey( + Species, models.DO_NOTHING, blank=True, null=True) + range = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) endemic = models.TextField(blank=True, null=True) source_url = models.TextField(blank=True, null=True) @@ -787,7 +834,8 @@ class SpeciesRange(models.Model): class PeopleRange(models.Model): id = models.AutoField(primary_key=True) person = models.ForeignKey(Person, models.CASCADE, blank=True, null=True) - range = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) + range = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) def __str__(self): return str(self.id) @@ -795,7 +843,8 @@ class PeopleRange(models.Model): class TaxonRange(models.Model): id = models.AutoField(primary_key=True) - range = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) + range = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) taxon = models.TextField(blank=True, null=True) subrange_or_region = models.TextField(blank=True, null=True) taxon_status = models.TextField(blank=True, null=True) @@ -826,7 +875,8 @@ class PeopleResource(models.Model): ) id = models.AutoField(primary_key=True) person = models.ForeignKey(Person, models.CASCADE, blank=True, null=True) - resource = models.ForeignKey(Resource, models.DO_NOTHING, blank=True, null=True) + resource = models.ForeignKey( + Resource, models.DO_NOTHING, blank=True, null=True) role = models.TextField(blank=True, null=True, choices=ROLES_CHOICES) def __str__(self): @@ -835,8 +885,10 @@ class PeopleResource(models.Model): class RangeCountry(models.Model): id = models.AutoField(primary_key=True) - range = models.ForeignKey(MountainRange, models.DO_NOTHING, blank=True, null=True) - country = models.ForeignKey(Country, models.DO_NOTHING, blank=True, null=True) + range = models.ForeignKey( + MountainRange, models.DO_NOTHING, blank=True, null=True) + country = models.ForeignKey( + Country, models.DO_NOTHING, blank=True, null=True) def __str__(self): return '%s --- %s' % (self.range.range_name if self.range else 'None', self.country.short_name if self.country else 'None') From 5446daa87fc1c7872f94f277262f56d926f01029 Mon Sep 17 00:00:00 2001 From: app Date: Mon, 19 Sep 2022 03:35:14 +0000 Subject: [PATCH 33/64] Use hardcoded instance name for now --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c709070..f3fc647 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,5 +1,5 @@ upstream gmba_django { - server 127.0.0.1:8000; + server gmba_django_nginx_1:8000; } server { From 5b64c4ee150ca4577f9c0da5637c6167f84fb0d2 Mon Sep 17 00:00:00 2001 From: app Date: Mon, 19 Sep 2022 03:57:24 +0000 Subject: [PATCH 34/64] Use correct container name --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index f3fc647..e644302 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,5 +1,5 @@ upstream gmba_django { - server gmba_django_nginx_1:8000; + server web:8000; } server { From a91d2b16b6c480871b7ad4c93139ead81dce03aa Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 19 Sep 2022 09:42:19 +0530 Subject: [PATCH 35/64] Fix static files path and create the directory --- Dockerfile.prod | 1 + nginx/nginx.conf | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/Dockerfile.prod b/Dockerfile.prod index e389e79..b169692 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -44,6 +44,7 @@ RUN addgroup -S app && adduser -S app -G app ENV HOME=/home/app ENV APP_HOME=/home/app/app RUN mkdir $APP_HOME +RUN mkdir $APP_HOME/staticfiles WORKDIR $APP_HOME # install dependencies diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e644302..93ba39f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -12,4 +12,9 @@ server { proxy_redirect off; } + + location /static/ { + alias /home/app/app/staticfiles/; + } + } From 13b7918de108219b2fd80f067c2f45853ddb3a75 Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 19 Sep 2022 09:54:17 +0530 Subject: [PATCH 36/64] Fix staticfiles path as per code --- Dockerfile.prod | 2 +- docker-compose.prod.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index b169692..6f7da8e 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -44,7 +44,7 @@ RUN addgroup -S app && adduser -S app -G app ENV HOME=/home/app ENV APP_HOME=/home/app/app RUN mkdir $APP_HOME -RUN mkdir $APP_HOME/staticfiles +RUN mkdir $APP_HOME/static WORKDIR $APP_HOME # install dependencies diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 8d6c176..b225f25 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -7,7 +7,7 @@ services: dockerfile: Dockerfile.prod command: gunicorn gmba_django.wsgi:application --bind 0.0.0.0:8000 volumes: - - static_volume:/home/app/app/staticfiles + - static_volume:/home/app/app/static expose: - 8000 env_file: @@ -23,7 +23,7 @@ services: nginx: build: ./nginx volumes: - - static_volume:/home/app/app/staticfiles + - static_volume:/home/app/app/static ports: - 1337:80 depends_on: From 600ae8e5de019bed08c7bde90e1840174122b04a Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 19 Sep 2022 09:57:57 +0530 Subject: [PATCH 37/64] Fix static files path in nginx.conf --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 93ba39f..3a25edf 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -14,7 +14,7 @@ server { location /static/ { - alias /home/app/app/staticfiles/; + alias /home/app/app/static/; } } From 4cc03b63725d2721c88ec45e57f8a7e9726b46b9 Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 19 Sep 2022 12:22:52 +0530 Subject: [PATCH 38/64] Remove unwanted files --- Dockerfile | 7 ------- docker-compose.yml | 22 ---------------------- 2 files changed, 29 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e730dbb..0000000 --- a/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3 -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -WORKDIR /code -COPY requirements.txt /code/ -RUN pip install -r requirements.txt -COPY . /code/ diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 96eae14..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '3.8' - -services: - web: - build: ./app - command: gunicorn gmba_django.wsgi:application --bind 0.0.0.0:8000 - ports: - - 8000:8000 - env_file: - - ./.env.prod - depends_on: - - db - db: - image: postgres:13.0-alpine - volumes: - - postgres_data:/var/lib/postgresql/data/ - env_file: - - ./.env.prod - -volumes: - postgres_data: - From bc4061f12a30c51497a9a05cfd428076a933dcfa Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 19 Sep 2022 12:36:21 +0530 Subject: [PATCH 39/64] Prepare changelog for 2.0 --- Changelog.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/Changelog.md b/Changelog.md index f099f2b..63d1722 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,46 @@ # CHANGELOG.md +## 2.0 (2022-09-19) Dockerify https://redmine.ungleich.ch/issues/10884 + +Steps: + - Create .env.prod and .env.prod.db files + +#### .env.prod +``` +DEBUG=0 +SECRET_KEY=secret_key_here +DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1] +DJANGO_SETTINGS_MODULE=gmba_django.settings.production +ALLOWED_HOSTS=localhost,127.0.0.1,[::1] +SQL_ENGINE=django.db.backends.postgresql +SQL_DATABASE=psql_db_name +SQL_USER=psql_username_name +SQL_PASSWORD=psql_passwordd +SQL_HOST=db +SQL_PORT=5432 +``` + +#### .env.prod.db +``` +POSTGRES_USER=psql_username_name +POSTGRES_PASSWORD=psql_passwordd +POSTGRES_DB=psql_db_name +``` + +Building containers + - docker-compose -f docker-compose.prod.yml down -v --remove-orphans + - docker-compose -f docker-compose.prod.yml up -d --build + +Copy db and adust + - docker cp gd.sql gmba_django_db_1:/gd.sql + - docker-compose exec db psql --username=psql_username_name --dbname=psql_db_name < /gd.sql + +Migrate / check + - docker-compose exec web python manage.py migrate --noinput + +Static resources + - docker-compose -f docker-compose.prod.yml exec web python manage.py collectstatic --no-input --clear + ## 1.12 (2022-01-19) Bugfixes: From 3eb5ba12d9423977a7193279ee5f269c802f3c44 Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 19 Sep 2022 13:30:56 +0530 Subject: [PATCH 40/64] Add missing configs for nginx --- nginx/nginx.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 3a25edf..75f3c54 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -5,6 +5,8 @@ upstream gmba_django { server { listen 80; + client_max_body_size 256m; + location / { proxy_pass http://gmba_django; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -12,9 +14,9 @@ server { proxy_redirect off; } - location /static/ { alias /home/app/app/static/; } + add_header Content-Security-Policy "frame-ancestors https://www.gmba.unibe.ch" } From 0123fd2eef17a52522bda912e8fd32aa9c9a5bd9 Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 19 Sep 2022 13:35:02 +0530 Subject: [PATCH 41/64] Add media files config --- Dockerfile.prod | 1 + docker-compose.prod.yml | 3 +++ nginx/nginx.conf | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/Dockerfile.prod b/Dockerfile.prod index 6f7da8e..408c26a 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -45,6 +45,7 @@ ENV HOME=/home/app ENV APP_HOME=/home/app/app RUN mkdir $APP_HOME RUN mkdir $APP_HOME/static +RUN mkdir $APP_HOME/media WORKDIR $APP_HOME # install dependencies diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b225f25..575027d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -8,6 +8,7 @@ services: command: gunicorn gmba_django.wsgi:application --bind 0.0.0.0:8000 volumes: - static_volume:/home/app/app/static + - media_volume:/home/app/app/media expose: - 8000 env_file: @@ -24,6 +25,7 @@ services: build: ./nginx volumes: - static_volume:/home/app/app/static + - media_volume:/home/app/app/media ports: - 1337:80 depends_on: @@ -32,3 +34,4 @@ services: volumes: postgres_data: static_volume: + media_volume: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 75f3c54..ceab716 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -16,6 +16,12 @@ server { location /static/ { alias /home/app/app/static/; + expires max; + } + + location /media/ { + alias /home/app/app/media/; + expires max; } add_header Content-Security-Policy "frame-ancestors https://www.gmba.unibe.ch" From e798679c6ca0824ac2ea4e4e6fd48e870932ba6b Mon Sep 17 00:00:00 2001 From: app Date: Tue, 20 Sep 2022 03:42:17 +0000 Subject: [PATCH 42/64] Change default db to psql --- gmba_django/settings/production.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gmba_django/settings/production.py b/gmba_django/settings/production.py index bcb825e..5ebe8eb 100644 --- a/gmba_django/settings/production.py +++ b/gmba_django/settings/production.py @@ -6,8 +6,8 @@ ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', "").split(",") DATABASES = { 'default': { - 'ENGINE': os.environ.get('SQL_ENGINE', 'django.db.backends.sqlite3'), - 'NAME': os.environ.get('SQL_DATABASE', os.path.join(BASE_DIR, 'db.sqlite3')), + 'ENGINE': os.environ.get('SQL_ENGINE', 'django.db.backends.postgresql'), + 'NAME': os.environ.get('SQL_DATABASE', 'app'), 'USER': os.environ.get('SQL_USER', 'app'), 'PASSWORD': os.environ.get('SQL_PASSWORD', ''), 'HOST': os.environ.get('SQL_HOST', 'localhost'), From 8f2f8b86e2b9bb3395cf2c33e5bbfcfcd6eaa859 Mon Sep 17 00:00:00 2001 From: app Date: Tue, 20 Sep 2022 03:48:07 +0000 Subject: [PATCH 43/64] Use POSTGRES_* constants for uniformity --- gmba_django/settings/production.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gmba_django/settings/production.py b/gmba_django/settings/production.py index 5ebe8eb..ae29e0a 100644 --- a/gmba_django/settings/production.py +++ b/gmba_django/settings/production.py @@ -7,9 +7,9 @@ ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', "").split(",") DATABASES = { 'default': { 'ENGINE': os.environ.get('SQL_ENGINE', 'django.db.backends.postgresql'), - 'NAME': os.environ.get('SQL_DATABASE', 'app'), - 'USER': os.environ.get('SQL_USER', 'app'), - 'PASSWORD': os.environ.get('SQL_PASSWORD', ''), + 'NAME': os.environ.get('POSTGRES_DB', 'app'), + 'USER': os.environ.get('POSTGRES_USER', 'app'), + 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', ''), 'HOST': os.environ.get('SQL_HOST', 'localhost'), 'PORT': os.environ.get('SQL_PORT', ''), } From 0140a656029c636e170b5fb49a85e25f4f6858f8 Mon Sep 17 00:00:00 2001 From: app Date: Tue, 20 Sep 2022 03:50:02 +0000 Subject: [PATCH 44/64] Align constants --- Changelog.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Changelog.md b/Changelog.md index 63d1722..b67106e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -9,13 +9,11 @@ Steps: ``` DEBUG=0 SECRET_KEY=secret_key_here -DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1] DJANGO_SETTINGS_MODULE=gmba_django.settings.production ALLOWED_HOSTS=localhost,127.0.0.1,[::1] -SQL_ENGINE=django.db.backends.postgresql -SQL_DATABASE=psql_db_name -SQL_USER=psql_username_name -SQL_PASSWORD=psql_passwordd +POSTGRES_USER=user +POSTGRES_PASSWORD=password +POSTGRES_DB=app SQL_HOST=db SQL_PORT=5432 ``` From 5e50d3320e1424cb074c9a6751d060b3951147d5 Mon Sep 17 00:00:00 2001 From: app Date: Tue, 20 Sep 2022 03:50:41 +0000 Subject: [PATCH 45/64] Use the same env file for db and web (.env.prod) --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b225f25..dd24b09 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -19,7 +19,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data/ env_file: - - ./.env.prod.db + - ./.env.prod nginx: build: ./nginx volumes: From 02aaccffedc0740a7372e9ce093863de5e903a2d Mon Sep 17 00:00:00 2001 From: app Date: Tue, 20 Sep 2022 04:02:47 +0000 Subject: [PATCH 46/64] Attempt collectstatic in Dockerfile itselfw --- Dockerfile.prod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.prod b/Dockerfile.prod index 6f7da8e..1e20d1d 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -67,6 +67,8 @@ RUN chown -R app:app $APP_HOME # change to the app user USER app +python gmba_django/manage.py collectstatic --noinput + # run entrypoint.prod.sh ENTRYPOINT ["/home/app/app/entrypoint.prod.sh"] From 854ce7599338cdd2172c62ff43cae44cf64e2e0b Mon Sep 17 00:00:00 2001 From: app Date: Tue, 20 Sep 2022 04:20:44 +0000 Subject: [PATCH 47/64] Fix location of manage.py --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 1e20d1d..b3a8ecc 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -67,7 +67,7 @@ RUN chown -R app:app $APP_HOME # change to the app user USER app -python gmba_django/manage.py collectstatic --noinput +CMD python manage.py collectstatic --noinput # run entrypoint.prod.sh ENTRYPOINT ["/home/app/app/entrypoint.prod.sh"] From 5326ea7ab094802de2807502f2e45af4c0c3d359 Mon Sep 17 00:00:00 2001 From: app Date: Tue, 20 Sep 2022 04:21:04 +0000 Subject: [PATCH 48/64] Use psql for dev as well --- gmba_django/settings/development.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gmba_django/settings/development.py b/gmba_django/settings/development.py index b9114a7..869ffde 100644 --- a/gmba_django/settings/development.py +++ b/gmba_django/settings/development.py @@ -4,7 +4,11 @@ DEBUG = True DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'ENGINE': os.environ.get('SQL_ENGINE', 'django.db.backends.postgresql'), + 'NAME': os.environ.get('POSTGRES_DB', 'app'), + 'USER': os.environ.get('POSTGRES_USER', 'app'), + 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', ''), + 'HOST': os.environ.get('SQL_HOST', 'localhost'), + 'PORT': os.environ.get('SQL_PORT', ''), } -} \ No newline at end of file +} From 3a5cbc6325b0bafa68e720e56ea2c9fc9c97c0dd Mon Sep 17 00:00:00 2001 From: app Date: Thu, 22 Sep 2022 03:06:24 +0000 Subject: [PATCH 49/64] Use POSTGRES_HOST and POSTGRES_PORT instead of SQL_* --- Changelog.md | 11 ++--------- gmba_django/settings/development.py | 4 ++-- gmba_django/settings/production.py | 4 ++-- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/Changelog.md b/Changelog.md index b67106e..a6cdbee 100644 --- a/Changelog.md +++ b/Changelog.md @@ -14,15 +14,8 @@ ALLOWED_HOSTS=localhost,127.0.0.1,[::1] POSTGRES_USER=user POSTGRES_PASSWORD=password POSTGRES_DB=app -SQL_HOST=db -SQL_PORT=5432 -``` - -#### .env.prod.db -``` -POSTGRES_USER=psql_username_name -POSTGRES_PASSWORD=psql_passwordd -POSTGRES_DB=psql_db_name +POSTGRES_HOST=db +POSTGRES_PORT=5432 ``` Building containers diff --git a/gmba_django/settings/development.py b/gmba_django/settings/development.py index 869ffde..bab09d0 100644 --- a/gmba_django/settings/development.py +++ b/gmba_django/settings/development.py @@ -8,7 +8,7 @@ DATABASES = { 'NAME': os.environ.get('POSTGRES_DB', 'app'), 'USER': os.environ.get('POSTGRES_USER', 'app'), 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', ''), - 'HOST': os.environ.get('SQL_HOST', 'localhost'), - 'PORT': os.environ.get('SQL_PORT', ''), + 'HOST': os.environ.get('POSTGRES_HOST', 'localhost'), + 'PORT': os.environ.get('POSTGRES_PORT', 5432), } } diff --git a/gmba_django/settings/production.py b/gmba_django/settings/production.py index ae29e0a..038ece5 100644 --- a/gmba_django/settings/production.py +++ b/gmba_django/settings/production.py @@ -10,8 +10,8 @@ DATABASES = { 'NAME': os.environ.get('POSTGRES_DB', 'app'), 'USER': os.environ.get('POSTGRES_USER', 'app'), 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', ''), - 'HOST': os.environ.get('SQL_HOST', 'localhost'), - 'PORT': os.environ.get('SQL_PORT', ''), + 'HOST': os.environ.get('POSTGRES_HOST', 'localhost'), + 'PORT': os.environ.get('POSTGRES_PORT', 5432), } } From 4734789a2518fdd8b1ec7c2ab08535851c398fd5 Mon Sep 17 00:00:00 2001 From: app Date: Thu, 22 Sep 2022 03:10:41 +0000 Subject: [PATCH 50/64] gunicorn: Listen on both ipv6 and ipv4 interfaces --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 8c19d5d..146a4ad 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -5,7 +5,7 @@ services: build: context: ./ dockerfile: Dockerfile.prod - command: gunicorn gmba_django.wsgi:application --bind 0.0.0.0:8000 + command: gunicorn gmba_django.wsgi:application --bind [::]:8000 volumes: - static_volume:/home/app/app/static - media_volume:/home/app/app/media From c7b525875da465a54c8295676f8b067a443fcf83 Mon Sep 17 00:00:00 2001 From: app Date: Thu, 22 Sep 2022 03:23:35 +0000 Subject: [PATCH 51/64] entrypoint.sh: Avoid using deep link --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index e87f0d1..391c292 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -71,5 +71,5 @@ USER app CMD python manage.py collectstatic --noinput # run entrypoint.prod.sh -ENTRYPOINT ["/home/app/app/entrypoint.prod.sh"] +ENTRYPOINT ["./entrypoint.prod.sh"] From 079762250bf7cb7a53d36cbc9ffdcd5d797d6e74 Mon Sep 17 00:00:00 2001 From: app Date: Thu, 22 Sep 2022 03:24:21 +0000 Subject: [PATCH 52/64] nginx.conf: add missing terminating semicolon --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index ceab716..f673c3f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -24,5 +24,5 @@ server { expires max; } - add_header Content-Security-Policy "frame-ancestors https://www.gmba.unibe.ch" + add_header Content-Security-Policy "frame-ancestors https://www.gmba.unibe.ch"; } From b7dfb676ec2ac292f29e3172f76b885f87a61292 Mon Sep 17 00:00:00 2001 From: app Date: Fri, 23 Sep 2022 01:29:27 +0000 Subject: [PATCH 53/64] Fix collectstatic --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 391c292..d0cf806 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -68,7 +68,7 @@ RUN chown -R app:app $APP_HOME # change to the app user USER app -CMD python manage.py collectstatic --noinput +RUN python manage.py collectstatic --noinput # run entrypoint.prod.sh ENTRYPOINT ["./entrypoint.prod.sh"] From 929d279b248c5ae90b9a7b9e2b496fdbe86f2ad5 Mon Sep 17 00:00:00 2001 From: PCoder Date: Fri, 23 Sep 2022 08:18:38 +0530 Subject: [PATCH 54/64] Use storage under /data --- Dockerfile.prod | 6 +++--- docker-compose.prod.yml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index d0cf806..45c9f34 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -35,14 +35,14 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/gmba_django/wheels - FROM python:3.9.6-alpine # create directory for the app user -RUN mkdir -p /home/app +RUN mkdir -p /data/app # create the app user RUN addgroup -S app && adduser -S app -G app # create the appropriate directories -ENV HOME=/home/app -ENV APP_HOME=/home/app/app +ENV HOME=/data/app +ENV APP_HOME=/data/app/app RUN mkdir $APP_HOME RUN mkdir $APP_HOME/static RUN mkdir $APP_HOME/media diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 146a4ad..0c75195 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -7,8 +7,8 @@ services: dockerfile: Dockerfile.prod command: gunicorn gmba_django.wsgi:application --bind [::]:8000 volumes: - - static_volume:/home/app/app/static - - media_volume:/home/app/app/media + - static_volume:/data/app/app/static + - media_volume:/data/app/app/media expose: - 8000 env_file: @@ -18,14 +18,14 @@ services: db: image: postgres:13.0-alpine volumes: - - postgres_data:/var/lib/postgresql/data/ + - postgres_data:/data/var/lib/postgresql/data/ env_file: - ./.env.prod nginx: build: ./nginx volumes: - - static_volume:/home/app/app/static - - media_volume:/home/app/app/media + - static_volume:/data/app/app/static + - media_volume:/data/app/app/media ports: - 1337:80 depends_on: From 2c870df36cac8687347a56c2146080326939a48f Mon Sep 17 00:00:00 2001 From: app Date: Fri, 23 Sep 2022 03:37:54 +0000 Subject: [PATCH 55/64] Add init.sh --- init.sh | 1 + 1 file changed, 1 insertion(+) create mode 100755 init.sh diff --git a/init.sh b/init.sh new file mode 100755 index 0000000..1a2354a --- /dev/null +++ b/init.sh @@ -0,0 +1 @@ +python manage.py migrate From 674ab5d2884228f5996ab28d77147821fd4af257 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 24 Sep 2022 16:45:41 +0530 Subject: [PATCH 56/64] Copy init.sh to appropriate path --- Dockerfile.prod | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile.prod b/Dockerfile.prod index 45c9f34..e04db25 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -56,12 +56,15 @@ RUN pip install --no-cache /wheels/* # copy entrypoint.prod.sh COPY ./entrypoint.prod.sh . +COPY ./init.sh /init.sh RUN sed -i 's/\r$//g' $APP_HOME/entrypoint.prod.sh RUN chmod +x $APP_HOME/entrypoint.prod.sh # copy project COPY . $APP_HOME + + # chown all the files to the app user RUN chown -R app:app $APP_HOME From 19b115434fd935f2b21b262eccb166198fcacbca Mon Sep 17 00:00:00 2001 From: app Date: Sat, 24 Sep 2022 11:51:49 +0000 Subject: [PATCH 57/64] ++init.sh --- init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init.sh b/init.sh index 1a2354a..f40decd 100755 --- a/init.sh +++ b/init.sh @@ -1 +1,3 @@ python manage.py migrate + +gunicorn gmba_django.wsgi:application --bind [::]:8000 From bfe6fe610a61eafe8182ed03801ff1d782e87104 Mon Sep 17 00:00:00 2001 From: app Date: Sat, 24 Sep 2022 13:45:40 +0000 Subject: [PATCH 58/64] Test: disable contents of init.sh --- init.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/init.sh b/init.sh index f40decd..17138bd 100755 --- a/init.sh +++ b/init.sh @@ -1,3 +1,4 @@ -python manage.py migrate - -gunicorn gmba_django.wsgi:application --bind [::]:8000 +#!/bin/sh +#python manage.py migrate +# +#gunicorn gmba_django.wsgi:application --bind [::]:8000 From f88a037edcbdd92acbdd315c97cc54d98de3b024 Mon Sep 17 00:00:00 2001 From: app Date: Sun, 25 Sep 2022 10:19:08 +0000 Subject: [PATCH 59/64] First working version --- Dockerfile.prod | 27 +++++++++++++++++---------- init.sh | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index e04db25..cda9ffd 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -35,18 +35,21 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/gmba_django/wheels - FROM python:3.9.6-alpine # create directory for the app user -RUN mkdir -p /data/app +RUN mkdir -p /home/app # create the app user RUN addgroup -S app && adduser -S app -G app # create the appropriate directories -ENV HOME=/data/app -ENV APP_HOME=/data/app/app +ENV HOME=/home/app +ENV APP_HOME=/home/app/gmba_django RUN mkdir $APP_HOME RUN mkdir $APP_HOME/static RUN mkdir $APP_HOME/media -WORKDIR $APP_HOME + +#ADD gmba_django /data/app + +#WORKDIR /data/app/gmba_django # install dependencies RUN apk update && apk add libpq @@ -55,15 +58,16 @@ COPY --from=builder /usr/src/gmba_django/requirements.txt . RUN pip install --no-cache /wheels/* # copy entrypoint.prod.sh -COPY ./entrypoint.prod.sh . +COPY ./entrypoint.prod.sh /entrypoint.prod.sh COPY ./init.sh /init.sh -RUN sed -i 's/\r$//g' $APP_HOME/entrypoint.prod.sh -RUN chmod +x $APP_HOME/entrypoint.prod.sh +RUN sed -i 's/\r$//g' /entrypoint.prod.sh +RUN chmod +x /entrypoint.prod.sh # copy project -COPY . $APP_HOME - +#COPY . $APP_HOME +ADD . /home/app/gmba_django +WORKDIR /home/app/gmba_django # chown all the files to the app user RUN chown -R app:app $APP_HOME @@ -74,5 +78,8 @@ USER app RUN python manage.py collectstatic --noinput # run entrypoint.prod.sh -ENTRYPOINT ["./entrypoint.prod.sh"] +ENTRYPOINT ["/entrypoint.prod.sh"] +EXPOSE 8000 + +CMD ["gunicorn", "--bind", "[::]:8000", "--chdir", "/home/app/gmba_django", "--workers", "3", "gmba_django.wsgi:application"] diff --git a/init.sh b/init.sh index 17138bd..a483ef3 100755 --- a/init.sh +++ b/init.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/ash #python manage.py migrate # #gunicorn gmba_django.wsgi:application --bind [::]:8000 From 0574694d3841dbdbe9a6a0ae6a1c72029c6f5407 Mon Sep 17 00:00:00 2001 From: app Date: Sun, 25 Sep 2022 14:20:51 +0000 Subject: [PATCH 60/64] Revert back --- Dockerfile.prod | 2 +- init.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index cda9ffd..a67b632 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -75,7 +75,7 @@ RUN chown -R app:app $APP_HOME # change to the app user USER app -RUN python manage.py collectstatic --noinput +#RUN python manage.py collectstatic --noinput # run entrypoint.prod.sh ENTRYPOINT ["/entrypoint.prod.sh"] diff --git a/init.sh b/init.sh index a483ef3..74b9cbd 100755 --- a/init.sh +++ b/init.sh @@ -1,4 +1,6 @@ #!/bin/ash #python manage.py migrate + +python manage.py collectstatic --noinput # #gunicorn gmba_django.wsgi:application --bind [::]:8000 From d7199827da8cdc1c407b1ae57130f272c31740fb Mon Sep 17 00:00:00 2001 From: PCoder Date: Sun, 25 Sep 2022 21:21:42 +0530 Subject: [PATCH 61/64] Fix infinite redirections in production --- gmba_django/settings/production.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gmba_django/settings/production.py b/gmba_django/settings/production.py index 038ece5..13ba57a 100644 --- a/gmba_django/settings/production.py +++ b/gmba_django/settings/production.py @@ -17,6 +17,9 @@ DATABASES = { SECURE_SSL_REDIRECT = True +# See https://docs.djangoproject.com/en/2.2/ref/settings/#secure-proxy-ssl-header +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + SESSION_COOKIE_SECURE = True CSRF_COOKIE_SECURE = True From 30f0743c4134c1a92c731831deaee33005bd6091 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 29 Sep 2022 07:40:52 +0530 Subject: [PATCH 62/64] #11013: filter persons only with profile_on_web true for public search --- app/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views.py b/app/views.py index 7198848..7c2f211 100644 --- a/app/views.py +++ b/app/views.py @@ -213,10 +213,10 @@ class SearchView(View): per_page = int(self.request.GET.get('per_page', 10)) q = self.request.GET.get('q', '').strip() if not q or len(q) < 3: - query_set = Person.objects.all() + query_set = Person.objects.filter(profile_on_web=True).all() else: query = reduce(operator.or_, (Q(field_indexer__icontains=item) for item in q.split(" "))) - query_set = Person.objects.filter(query) + query_set = Person.objects.filter(query).filter(profile_on_web=True) q_country = self.request.GET.get('country', '') q_range = self.request.GET.get('range', '') From 3b30b9666a19b1aa2fa848b1561c4012bfdbfea8 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 29 Sep 2022 08:58:00 +0530 Subject: [PATCH 63/64] Update Changlog --- Changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Changelog.md b/Changelog.md index a6cdbee..da1cfc5 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ # CHANGELOG.md +## 2.1 (2022-09-29) +- issue#11013: non-public entries appearing as public + ## 2.0 (2022-09-19) Dockerify https://redmine.ungleich.ch/issues/10884 Steps: From 44bab85ba70874de1f46fc965f5d0a29fbec9e28 Mon Sep 17 00:00:00 2001 From: Sami Date: Tue, 4 Oct 2022 19:21:57 +0500 Subject: [PATCH 64/64] person deletion method overload --- app/admin.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 2 deletions(-) diff --git a/app/admin.py b/app/admin.py index 80ae861..5df424f 100644 --- a/app/admin.py +++ b/app/admin.py @@ -9,6 +9,7 @@ from django.http import HttpResponse import csv from django import forms from django.contrib.admin.widgets import AutocompleteSelect +from django.contrib import messages # import messages class autocomplete_fields_form(forms.ModelForm): @@ -124,6 +125,81 @@ class PersonAdmin(admin.ModelAdmin): 'country', 'organisation_english', 'organisation_2', 'organisation_3', 'acronym') return queryset + def get_actions(self, request): + actions = super().get_actions(request) + if 'delete_selected' in actions: + del actions['delete_selected'] + return actions + + def delete_model(self, request, obj): + self.delete_person(request, obj) + return super(PersonAdmin, self).delete_model(request, obj) + + def delete_person(self, request, obj): + taxa_people = TaxaPeople.objects.filter(person=obj.id) + for i in taxa_people: + i.delete() + ranges_people = RangesPeople.objects.filter(person=obj.id) + for i in ranges_people: + i.delete() + resources_people = ResourcesPeople.objects.filter(person=obj.id) + for i in resources_people: + i.delete() + scales_people = ScalesPeople.objects.filter(person=obj.id) + for i in scales_people: + i.delete() + people_function = PeopleFunction.objects.filter(person=obj.id) + for i in people_function: + i.delete() + people_range = PeopleRange.objects.filter(person=obj.id) + for i in people_range: + i.delete() + people_resources = PeopleResource.objects.filter(person=obj.id) + for i in people_resources: + i.delete() + fields_people = FieldsPeople.objects.filter(person=obj.id) + for i in fields_people: + i.delete() + methods_people = MethodsPeople.objects.filter(person=obj.id) + for i in methods_people: + i.delete() + + def delete_persons(self, request, obj): + for o in obj.all(): + taxa_people = TaxaPeople.objects.filter(person=o.id) + for i in taxa_people: + i.delete() + ranges_people = RangesPeople.objects.filter(person=o.id) + for i in ranges_people: + i.delete() + resources_people = ResourcesPeople.objects.filter(person=o.id) + for i in resources_people: + i.delete() + scales_people = ScalesPeople.objects.filter(person=o.id) + for i in scales_people: + i.delete() + people_function = PeopleFunction.objects.filter(person=o.id) + for i in people_function: + i.delete() + people_range = PeopleRange.objects.filter(person=o.id) + for i in people_range: + i.delete() + people_resources = PeopleResource.objects.filter(person=o.id) + for i in people_resources: + i.delete() + fields_people = FieldsPeople.objects.filter(person=o.id) + for i in fields_people: + i.delete() + methods_people = MethodsPeople.objects.filter(person=o.id) + for i in methods_people: + i.delete() + o.delete() + messages.success(request, "Successfully deleted") + + delete_persons.short_description = 'Delete Person & Related Fields' + + actions = [download_csv, delete_persons] + class ResourceKeywordInline(admin.TabularInline): autocomplete_fields = ['keyword'] @@ -192,7 +268,26 @@ class ResourceAdmin(admin.ModelAdmin): del actions['delete_selected'] return actions + def delete_model(self, request, obj): + self.delete_resource(request, obj) + return super(ResourceAdmin, self).delete_model(request, obj) + def delete_resource(self, request, obj): + resource_keyword = ResourceKeyword.objects.filter(resource=obj.id) + for i in resource_keyword: + i.delete() + resource_range = ResourceRange.objects.filter(resource_title=obj.id) + for i in resource_range: + i.delete() + resource_people = ResourcesPeople.objects.filter(resource=obj.id) + for i in resource_people: + i.delete() + resource_people_resource = PeopleResource.objects.filter( + resource=obj.id) + for i in resource_people_resource: + i.delete() + + def delete_resources(self, request, obj): for o in obj.all(): resource_keyword = ResourceKeyword.objects.filter(resource=o.id) for i in resource_keyword: @@ -208,9 +303,10 @@ class ResourceAdmin(admin.ModelAdmin): for i in resource_people_resource: i.delete() o.delete() - delete_resource.short_description = 'Delete Resource & Related Fields' + messages.success(request, "Successfully deleted") + delete_resources.short_description = 'Delete Resource & Related Fields' - actions = [download_csv, delete_resource] + actions = [download_csv, delete_resources] class RangeNameTranslationInline(admin.TabularInline):