import django.db.utils from django.core.management.base import BaseCommand from django.apps import apps from django.db.utils import IntegrityError from app.models import * import csv import json class Command(BaseCommand): help = 'Imports csv to DB' csv_files_models_dict = { "v2-LU_GMBA_SpeciesGroups.csv": "GMBA_SpeciesGroup", "v2-LU_Countries.csv": "Country", "v2-LU_Languages.csv": "Language", "v2-LU_Sources.csv": "Source", "v2-LU_RedListCategories.csv": "RedListCategory", "v2-LU_RangeTypes.csv": "RangeType", "v2-LU_PeopleStatus.csv": "PeopleStatus", "v2-LU_TrendsQuantity.csv": "TrendsQuantity", "v2-LU_TrendsQuality.csv": "TrendsQuality", "v2-LU_TaxonUnit.csv": "TaxonUnit", "v2-LU_TaxonStatus.csv": "TaxonStatus", "v2-Ranges-cleaned.csv": "MountainRange", "v2-AddElevations.csv": "AddElevation", "v2-GMBA_Function.csv": "GMBA_function", "v2-Gmba_V2_centroid.csv": "GMBA_V2_Centroid", "v2-ImportGeom210915.csv": "ImportGeom210915", "v2-LanguageLink.csv": "LanguageLink", "v2-Keywords.csv": "Keyword", "v2-NamesImport.csv": "NamesImport", "v2-Organisations-cleaned.csv": "Organisation", "v2-Peaks.csv": "Peak", "v2-People.csv": "Person", "v2-PeopleRanges.csv": "PeopleRange", "v2-PeopleFunction.csv": "PeopleFunction", "v2-Resources.csv": "Resource", "v2-PeopleResources.csv": "PeopleResource", "v2-RangeCountries.csv": "RangeCountry", "v2-RangeNameTranslations.csv": "RangeNameTranslation", "v2-RangeOnlineInfo.csv": "RangeOnlineInfo", "v2-ResourceRanges.csv": "ResourceRange", "v2-ResourceKeywords.csv": "ResourceKeyword", "v2-Repositories.csv": "Repository", "v2-Species.csv": "Species", "v2-Searches.csv": "Search", "v2-TaxonRange.csv": "TaxonRange", "v2-SpeciesRange.csv": "SpeciesRange" } cols_to_django_fields = { "ID": 'id', "Source": 'source', "RangeName": 'range_name_id', "LanguageTranslation": 'language_translation_id', "RangeNameTranslation": 'range_name_translation', "GMBA_ID_v2": 'gmba_v2_id', "Elev_Min": 'elev_min', "Elev_Max": 'elev_max', "Elev_Range": 'elev_range', "TaxonStatus": 'taxon_status', "InfoSource": 'info_source', "URL": 'url', "GMBA function": 'gmba_function', "TaxonUnit": 'taxon_unit', "Range_ID": 'id', "RangeNameMap": 'range_name_map', "RangeNameAscii": 'range_name_ascii', "RangeNameLanguage": 'range_name_language', "MotherRange": 'mother_range', "Feature": 'feature', "MapUnit": 'map_unit', "Level": 'level', "LevelText": 'level_text', "Level_1": 'level_1', "Level_2": 'level_2', "Level_3": 'level_3', "Latitude": 'latitude', "Longitude": 'longitude', "Orogeny": 'orogeny', "Area": 'area', "GMBA_V1_ID": 'GMBA_v1_id', "Countries": 'countries', "Peak_Elevation": 'peak_elevation', "Peak_Name": 'peak_name', "Peak_Latitude": 'peak_latitude', "Peak_Longitude": 'peak_longitude', "Comments": 'comments', "Checked": 'checked', "Range_AlternateID": 'range_alternate_id', "GeologicRegion": 'geologic_region', "GMBA_V2_ID": 'gmba_v2_id', "GMBA_V2_ID_str": 'gmba_v2_id_str', "WikiDataID": 'wiki_data_id', "WikiDataURL": 'wiki_data_url', "Select_300": 'select_300', "Gmba_Narrow": 'gmba_narrow', "Name_FR": 'name_fr', "Name_DE": 'name_de', "Name_ES": 'name_es', "Name_PT": 'name_pt', "Name_CN": 'name_cn', "Name_RU": 'name_ru', "Name_TR": 'name_tr', "Perimeter": 'perimeter', "ColorAll": 'color_all', "ColorBasic": 'color_basic', "Color300": 'color_300', "Elev_Low": 'elev_low', "Elev_High": 'elev_high', "Elev_Avg": 'elev_avg', "gridcode": 'gridcode', "Trend": 'trend', "RepositoryName": 'repository_name', "RepositoryURL": 'repository_url', "Resource": 'resource_id', "Keyword": 'keyword', "Keyword_ID": 'keyword_id', "Mother": 'mother', "CN": 'cn', "DE": 'de', "ES": 'es', "FR": 'fr', "PT": 'pt', "RU": 'ru', "TR": 'tr', "ResourceTitle": 'resource_title_id', "LanguageLetterCode": 'language_letter_code', "LanguageNumberCode": 'language_number_code_id', "OrgNum1": 'org_num1', "Organisation Search": 'organisation_search', "OrgAlphaSearch": 'org_alpha_search', "Organisation English": 'organisation_english', "Organisation 2": 'organisation_2', "Organisation 3": 'organisation_3', "Organisation Original": 'organisation_original', "Acronym": 'acronym', "Street": 'street', "PO Box": 'po_box', "Postcode": 'postcode', "City": 'city', "Region": 'region', "SearchURL": 'search_url', "LatLon": 'lat_long', "URL Org": 'url', "Tel Org": 'tel', "Email Org": 'email', "Country": 'country_id', "Tags": 'tags', "Description": 'description', "Northing": 'northing', "Easting": 'easting', "Category": 'category', "Subject": 'subject', "Title": 'title', "Citation": 'citation', "Type": 'type', "Abstract": 'abstract', "AuthorKeywords": 'author_keywords', "Lat": 'lat', "Lon": 'lon', "Stars": 'stars', "PEGASuS_Check_map_with_author": 'PEGASuS_Check_map_with_author', "PEGASuS_polygon_ID": 'PEGASuS_polygon_ID', "PEGASuS_Polygon_comments": 'PEGASuS_Polygon_comments', "PEGASuS_Assessment_ID": 'PEGASuS_Assessment_ID', "GLORIA": 'gloria', "GNOMO": 'gnomo', "LTER": 'lter', "LTSER": 'ltser', "MIREN": 'miren', "TEAM": 'team', "Inventory": 'inventory', "DOI": 'doi', "ShortName": 'short_name', "FormalName": 'formal_name', "Membership within the UN System": 'membership_within_un_system', "Membership within the UN System": 'membership_within_un_system', "Continent": 'continent', "EU_MS": 'eu_ms', "EEA_MS": 'eea_ms', "ISO3": 'iso3', "ISO2": 'iso2', "Point_Name": 'point_name', "Elevation": 'elevation', "Link": 'link', "Repository": 'repository_id', "SearchString": 'search_string', "SearchDate": 'search_date', "Result": 'result', "NumberOfRecords": 'number_of_records', "Stored": 'stored', "SpeciesGroup": 'species_group', "MrMrs": 'mr_mrs', "First name": 'first_name', "Last name": 'last_name', "Full name": 'full_name', "SearchName": 'search_name', "e-mail 1": 'contact_email', "e-mail 2": 'email_2', "Skype": 'skype', "Professional phone": 'professional_phone', "Mobile number": 'mobile_number', "Field of expertise": 'field_of_expertise', "Biography": 'biography', "Position": 'position', "Status": 'status', "Entry date": 'entry_date', "Newsletter": 'news_letter', "CountryLookup": 'country_lookup', "Organisation": 'organization_id', "Birds": 'birds', "Mammals": 'mammals', 'Reptiles': 'reptiles', 'Amphibians': 'amphibians', 'Fish': 'fish', 'Insects': 'insects', 'Molluscs': 'molluscs', 'Crustaceans': 'crustaceans', 'Arachnids': 'arachnids', 'Angiosperms': 'angiosperms', 'Gymnosperms': 'gymnosperms', 'Fungi': 'fungi', 'Algae': 'algae', 'Microbes': 'microbes', 'Biological field sampling': 'biological_field_sampling', 'Data mining': 'data_mining', 'Remote sensing': 'remote_sensing', 'GIS': 'gis', 'Spatial analysis': 'spatial_analysis', 'Statistical analysis': 'statistical_analysis', 'Modelling': 'modelling', 'Assessment': 'assessment', 'Meta-analysis': 'meta_analysis', 'Synthesis': 'synthesis', 'Qualitative social science methods (interviews, surveys)': 'qualitative_ssm', 'Genetic analyses': 'genetic_analyses', 'Field site': 'field_site', 'Transect': 'transect', 'Mountain top': 'mountain_top', 'Mountain range': 'mountain_range', 'Landscape': 'landscape', 'Regional': 'regional', 'National': 'national', 'Global': '_global', 'Geographic area of expertise': 'geographic_area_of_expertise', 'ProfileOnWeb': 'profile_on_web', 'Updated': 'updated', 'ORCID': 'orcid', 'WebOfScience': 'web_of_science', 'Twitter': 'twitter', 'Instagram': 'instagram', 'ScientificName': 'scientific_name_id', 'Class': '_class', 'EnglishName': 'english_name', 'Language': 'language', 'Person': 'person_id', 'Field': 'field_id', 'Method': 'method_id', 'Scale': 'scale_id', 'Function': 'function_id', 'Range': 'range_id', 'Endemic': 'endemic', 'SourceURL': 'source_url', 'MountainRange': 'mountain_range', 'TaxonRangeID': 'id', 'SubRangeOrRegion': 'subrange_or_region', 'Taxon': 'taxon_id', 'Distribution': 'distribution', 'RedList': 'redlist', 'CountUnit': 'count_unit', 'NumberUnits': 'number_of_units', 'Remarks': 'remarks', 'RangeType': 'range_type', 'Role': 'role', 'RedListCategory': 'red_list_category' } def add_arguments(self, parser): parser.add_argument('--path', type=str, help="file path") parser.add_argument('--csv_folder_path', type=str, help="Path where the csvs are located") parser.add_argument('--model_name', type=str, help="model name") parser.add_argument('--app_name', type=str, help="django app name that the model is connected to", default='app') parser.add_argument('--mother_range_reload', action='store_true', help="Whether we are trying to reload mother " "range relationship") parser.add_argument('--debug', action='store_true', help="Whether we want to debug") parser.add_argument('--all', action='store_true', help="Imports all csvs") # ./manage.py import --path /home/pcoder/Downloads/gmbadb/csvs/v2-LU_RedListCategories.csv --model_name RedListCategory --app_name app def handle(self, *args, **options): csv.register_dialect( 'mydialect', delimiter=',', quotechar='"', doublequote=True, skipinitialspace=True, lineterminator='\n', quoting=csv.QUOTE_MINIMAL) csv_folder_path = '/home/pcoder/Downloads/gmbadb/csvs' if options['csv_folder_path']: csv_folder_path = options['csv_folder_path'] debug = options.get('debug', None) if options.get('all'): print("Doing an import of all csvs") for csv_file_name, model_name in self.csv_files_models_dict.items(): print("Importing %s -- %s" % (csv_file_name, model_name)) models_to_ignore = ['Range', 'NamesImport', 'ImportGeom210915', 'Organization', 'AddElevation', 'GMBA_V2_Centroid', 'Person', 'PeopleRange', 'PeopleFunction', "PeopleResource", "RangeCountry", "RangeNameTranslation", "RangeOnlineInfo", "ResourceRange", "ResourceKeyword", "Repository"] models_to_ignore = [] if model_name in models_to_ignore: # we have already imported and do not want to spend more time redoing stuff continue if csv_folder_path.endswith('/'): file_path = '%s%s' % (csv_folder_path, csv_file_name) else: file_path = '%s/%s' % (csv_folder_path, csv_file_name) _model = apps.get_model(options.get('app_name', 'app'), model_name) with open(file_path, 'r') as csv_file: reader = csv.reader(csv_file, dialect='mydialect') first = True for row in reader: if first: # Assume the first row to be the header header = row header = [h.strip('"') for h in header] first = False continue _object_dict = {str(self.cols_to_django_fields.get(key)): str(value.lstrip('"').rstrip('"')) for key, value in zip(header, row)} _object_dict = handle_object_dict(_object_dict, model_name) m = _model(**_object_dict) try: m.save() except IntegrityError as ie: print(str(ie)) if "UNIQUE constraint failed: range.gmba_v2_id" in str(ie): print("======") print("Could not save %s" % json.dumps(_object_dict)) print("======") print("Done importing %s" % model_name) else: _model = apps.get_model(options.get('app_name', 'app'), options['model_name']) model_name = options['model_name'] mother_range_reload = options.get('mother_range_reload') k = '' csv_file_name = '' for k, v in self.csv_files_models_dict.items(): if v.strip().lower() == model_name.strip().lower(): csv_file_name = k if csv_file_name == '': raise Exception('Could not find a csv file name for model %s' % model_name) if csv_folder_path.endswith('/'): file_path = '%s%s' % (csv_folder_path, csv_file_name) else: file_path = '%s/%s' % (csv_folder_path, csv_file_name) csv.register_dialect( 'mydialect', delimiter=',', quotechar='"', doublequote=True, skipinitialspace=True, lineterminator='\n', quoting=csv.QUOTE_MINIMAL) with open(file_path, 'r', newline='') as csv_file: reader = csv.reader(csv_file, dialect='mydialect') first = True error_row_count = 0 total = 0 for row in reader: total += 1 if first: # Assume the first row to be the header header = row header = [h.strip('"') for h in header] first = False continue if mother_range_reload: # We have loaded the range model already and we are attempting to construct the mother_range # relationship try: this_range = MountainRange.objects.get(id=int(row[0])) r = MountainRange.objects.get(id=int(row[5])) # default Range which exists already this_range.mother_range = r this_range.save() except MountainRange.DoesNotExist as dne: print("this range = %s, mother range --> %s" % (row[0], row[5])) print(str(dne)) error_row_count += 1 except ValueError as ve: print('0 = %s and 5 = %s' % (row[0], row[5])) print(str(ve)) error_row_count += 1 continue _object_dict = {self.cols_to_django_fields.get(key): value.lstrip('"').rstrip('"') for key, value in zip(header, row)} _object_dict = handle_object_dict(_object_dict, model_name, debug=debug) try: m = _model(**_object_dict) m.save() except django.db.utils.IntegrityError as ex1: error_row_count += 1 print(str(ex1)) print("**********************") print(str(_object_dict)) print("**********************") except Exception as ex: error_row_count += 1 print('-----') print(str(ex)) print('----------------') continue print("Done importing %s" % str(_model)) print("Total rows = %s, error rows = %s" % (total, error_row_count)) def handle_object_dict(object_dict, model_name, debug=False): if model_name == 'Resource': object_dict['url'] = object_dict['url'].strip("#") for i in ['PEGASuS_Check_map_with_author', 'gloria', 'gnomo', 'lter', 'ltser', 'miren', 'team', 'inventory']: if i in object_dict: object_dict[i] = True if object_dict[i].lower().strip() == 'true' else False if model_name == 'MountainRange': # Reinstate range_name key object_dict['range_name'] = object_dict['range_name_id'] object_dict.pop('range_name_id') connected_fields = [{'range_name_language': Language}, {'feature': RangeType}] for f in connected_fields: for k, v in f.items(): if k in object_dict: if debug: print('Getting %s of %s' % (k, object_dict[k])) if object_dict[k] == '': object_dict[k] = 0 object_dict[k] = v.objects.get(id=int(object_dict[k])) if 'mother_range' in object_dict: if debug: print('Getting mother_range of %s' % object_dict['mother_range']) if object_dict['mother_range'] == '': object_dict['mother_range'] = 0 try: object_dict['mother_range'] = MountainRange.objects.get(id=0) # default Range which exists already except MountainRange.DoesNotExist as dne: print(str(dne)) for i in ['checked']: if i in object_dict: object_dict[i] = True if object_dict[i].lower().strip() == 'true' else False for i in ['select_300']: if i in object_dict: object_dict[i] = True if object_dict[i].lower().strip() == 'x' else False for i in ['gmba_narrow']: if i in object_dict: object_dict[i] = True if object_dict[i].lower().strip() == 'x' else False # area field can't be empty if 'area' in object_dict: if object_dict['area'] == '': object_dict['area'] = -1 if model_name == 'Keyword': if object_dict['mother'] == '': object_dict['mother'] = 0 try: object_dict['mother'] = Keyword.objects.get(keyword_id=object_dict['mother']) # default Keyword which exists already except Keyword.DoesNotExist as dne: object_dict['mother'] = Keyword.objects.get(keyword_id=0) print(str(dne)) if model_name == 'Organization' and 'country_id' in object_dict: object_dict['country'] = object_dict['country_id'] object_dict.pop('country_id') if model_name == 'PeopleRange' and 'mountain_range' in object_dict: object_dict['range_id'] = object_dict['mountain_range'] object_dict.pop('mountain_range') if model_name == 'Species' and 'scientific_name_id' in object_dict: object_dict['scientific_name'] = object_dict['scientific_name_id'] object_dict.pop('scientific_name_id') if model_name == 'TaxonRange' and 'taxon_id' in object_dict: object_dict['taxon'] = object_dict['taxon_id'] object_dict.pop('taxon_id') if model_name == 'Person' and 'organization_id' in object_dict: if debug: print("organization_id=%s" % object_dict['organization_id']) if object_dict['organization_id'] == '' or object_dict['organization_id'] is None: object_dict['organization_id'] = '-1' else: object_dict['organization_id'] = int(float(object_dict['organization_id'])) if 'status' in object_dict: if debug: print('Getting status of %s' % object_dict['status']) if object_dict['status'] == '': object_dict['status'] = 0 object_dict['status'] = PeopleStatus.objects.get(id=int(object_dict['status'])) if 'country_lookup' in object_dict: if object_dict['country_lookup'].strip() == '' or object_dict['country_lookup'] is None: object_dict['country_lookup'] = 0 if debug: print('Getting country of %s' % object_dict['country_lookup']) object_dict['country'] = Country.objects.get(id=int(object_dict['country_lookup'])) object_dict.pop('country_lookup') for i in ['news_letter', 'birds', 'mammals', 'reptiles', 'amphibians', 'fish', 'insects', 'molluscs', 'crustaceans', 'arachnids', 'angiosperms', 'gymnosperms', 'fungi', 'algae', 'microbes', 'biological_field_sampling', 'data_mining', 'remote_sensing', 'gis', 'spatial_analysis', 'statistical_analysis', 'modelling', 'assessment', 'meta_analysis', 'synthesis', 'qualitative_ssm', 'genetic_analyses', 'field_site', 'transect', 'mountain_top', 'mountain_range', 'landscape', 'regional', 'national', '_global', 'profile_on_web', 'updated']: if i in object_dict: object_dict[i] = True if object_dict[i].lower().strip() == 'true' else False if object_dict is None: if debug: print("Object None for %s" % model_name) # else: # print(object_dict) if str(object_dict) is not None else "Str object dict is None" return object_dict