From ec5f4111ac4c3902826c79d43e0ecdb654fd0a7e Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 8 Nov 2021 06:37:32 +0530 Subject: [PATCH] Add basic csv importer To test: ./manage.py import --path /home/pcoder/Downloads/gmbadb/csvs/v2-LU_RedListCategories.csv --model_name RedListCategory --app_name app --- app/management/commands/import.py | 291 ++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 app/management/commands/import.py diff --git a/app/management/commands/import.py b/app/management/commands/import.py new file mode 100644 index 0000000..b2cc12e --- /dev/null +++ b/app/management/commands/import.py @@ -0,0 +1,291 @@ +from django.core.management.base import BaseCommand, CommandError +from django.apps import apps + +from app.models import Country +import csv + +class Command(BaseCommand): + help = 'Imports csv to DB' + csv_files = [ + "v2-LU_GMBA_SpeciesGroups.csv", + "v2-LU_Countries.csv", + "v2-LU_Languages.csv", + "v2-LU_Sources.csv", + "v2-LU_RedListCategories.csv", + "v2-LU_RangeTypes.csv", + "v2-LU_PeopleStatus.csv", + "v2-LU_TrendsQuantity.csv", + "v2-LU_TrendsQuality.csv", + "v2-LU_TaxonUnit.csv", + "v2-LU_TaxonStatus.csv", + "v2-AddElevations.csv", + "v2-GMBA_Function.csv", + "v2-Gmba_V2_centroid.csv", + "v2-ImportGeom210915.csv", + "v2-LanguageLink.csv", + "v2-Keywords.csv", + "v2-NamesImport.csv", + "v2-Organisations.csv", + "v2-Peaks.csv", + "v2-PeopleRanges.csv", + "v2-PeopleFunction.csv", + "v2-PeopleResources.csv", + "v2-RangeCountries.csv", + "v2-RangeNameTranslations.csv", + "v2-RangeOnlineInfo.csv", + "v2-Ranges.csv", + "v2-ResourceRanges.csv", + "v2-ResourceKeywords.csv", + "v2-Repositories.csv", + "v2-Resources.csv", + "v2-Species.csv", + "v2-Searches.csv", + "v2-TaxonRange.csv", + "v2-SpeciesRange.csv", + "v2-People.csv" + ] + cols_to_django_fields = { + "ID": 'id', + "Source": 'source', + "RangeName": 'range_name', + "LanguageTranslation": 'language_translation', + "RangeNameTranslation": 'range_name_translation', + "GMBA_ID_v2": 'gmba_v2_id', + "Elev_Min": 'elev_min', + "Elev_Max": 'elev_max', + "Elev_Range": 'elev_range', + "TaxonStatus": 'taxon_status', + "InfoSource": 'info_source', + "URL": 'url', + "GMBA function": 'GMBA_function', + "TaxonUnit": 'taxon_unit', + "Range_ID": 'id', + "RangeNameMap": 'range_name_map', + "RangeNameAscii": 'range_name_ascii', + "RangeNameLanguage": 'range_name_language', + "MotherRange": 'mother_range', + "Feature": 'feature', + "MapUnit": 'map_unit', + "Level": 'level', + "LevelText": 'level_text', + "Level_1": 'level_1', + "Level_2": 'level_2', + "Level_3": 'level_3', + "Latitude": 'latitude', + "Longitude": 'longitude', + "Orogeny": 'orogeny', + "Area": 'area', + "GMBA_V1_ID": 'GMBA_v1_id', + "Countries": 'countries', + "Peak_Elevation": 'peak_elevation', + "Peak_Name": 'peak_name', + "Peak_Latitude": 'peak_latitude', + "Peak_Longitude": 'peak_longitude', + "Comments": 'comments', + "Checked": 'checked', + "Range_AlternateID": 'range_alternate_id', + "GeologicRegion": 'geologic_region', + "GMBA_V2_ID": 'gmba_v2_id', + "GMBA_V2_ID_str": 'gmba_v2_id_str', + "WikiDataID": 'wiki_data_id', + "WikiDataURL": 'wiki_data_url', + "Select_300": 'select_300', + "Gmba_Narrow": 'gmba_narrow', + "Name_FR": 'name_fr', + "Name_DE": 'name_de', + "Name_ES": 'name_es', + "Name_PT": 'name_pt', + "Name_CN": 'name_cn', + "Name_RU": 'name_ru', + "Name_TR": 'name_tr', + "Perimeter": 'perimeter', + "ColorAll": 'color_all', + "ColorBasic": 'color_basic', + "Color300": 'color_300', + "Elev_Low": 'elev_low', + "Elev_High": 'elev_high', + "Elev_Avg": 'elev_avg', + "gridcode": 'gridcode', + "Trend": 'trend', + "RepositoryName": 'repository_name', + "RepositoryURL": 'repository_url', + "Resource": 'resource', + "Keyword": 'keyword', + "Keyword_ID": 'keyword_id', + "Mother": 'mother', + "CN": 'cn', + "DE": 'de', + "ES": 'es', + "FR": 'fr', + "PT": 'pt', + "RU": 'ru', + "TR": 'tr', + "ResourceTitle": 'resource_title', + "LanguageLetterCode": 'language_letter_code', + "LanguageNumberCode": 'language_number_code', + "OrgNum1": 'org_num1', + "Organisation Search": 'organisation_search', + "OrgAlphaSearch": 'org_alpha_search', + "Organisation English": 'organisation_english', + "Organisation 2": 'organisation_2', + "Organisation 3": 'organisation_3', + "Organisation Original": 'organisation_original', + "Acronym": 'acronym', + "Street": 'street', + "PO Box": 'po_box', + "Postcode": 'postcode', + "City": 'city', + "Region": 'region', + "SearchURL": 'search_url', + "LatLon": 'lat_lon', + "URL Org": 'url', + "Tel Org": 'tel', + "Email Org": 'email', + "Country": 'country', + "Tags": 'tags', + "Description": 'description', + "Northing": 'northing', + "Easting": 'easting', + "Category": 'category', + "Subject": 'subject', + "Title": 'title', + "Citation": 'citation', + "Type": 'type', + "Abstract": 'abstract', + "AuthorKeywords": 'author_keywords', + "Lat": 'lat', + "Lon": 'lon', + "Stars": 'stars', + "PEGASuS_Check_map_with_author": 'PEGASuS_Check_map_with_author', + "PEGASuS_polygon_ID": 'PEGASuS_polygon_ID', + "PEGASuS_Polygon_comments": 'PEGASuS_Polygon_comments', + "PEGASuS_Assessment_ID": 'PEGASuS_Assessment_ID', + "GLORIA": 'gloria', + "GNOMO": 'gnomo', + "LTER": 'lter', + "LTSER": 'ltser', + "MIREN": 'miren', + "TEAM": 'team', + "Inventory": 'inventory', + "DOI": 'doi', + "ShortName": 'short_name', + "FormalName": 'formal_name', + "Membership within the UN System": '', + "Continent": 'continent', + "EU_MS": 'eu_ms', + "EEA_MS": 'eea_ms', + "ISO3": 'iso3', + "ISO2": 'iso2', + "Point_Name": 'point_name', + "Elevation": 'elevation', + "Link": 'link', + "Repository": 'repository', + "SearchString": 'search_string', + "SearchDate": 'search_date', + "Result": 'result', + "NumberOfRecords": 'number_of_records', + "Stored": 'stored', + "SpeciesGroup": 'species_group', + "MrMrs": 'mr_mrs', + "First name": 'first_name', + "Last name": 'last_name', + "Full name": 'full_name', + "SearchName": 'search_name', + "e-mail 1": 'email_1', + "e-mail 2": 'email_2', + "Skype": 'skype', + "Professional phone": 'professional_phone', + "Mobile number": 'mobile_number', + "Field of expertise": 'field_of_expertise', + "Biography": 'biography', + "Position": 'position', + "Status": 'status', + "Entry date": 'entry_date', + "Newsletter": 'news_letter', + "CountryLookup": 'country_lookup', + "Organisation": 'organization', + "Birds": 'birds', + "Mammals": 'mammals', + 'Reptiles': 'reptiles', + 'Amphibians': 'amphibians', + 'Fish': 'fish', + 'Insects': 'insects', + 'Molluscs': 'molluscs', + 'Crustaceans': 'crustaceans', + 'Arachnids': 'arachnids', + 'Angiosperms': 'angiosperms', + 'Gymnosperms': 'gymnosperms', + 'Fungi': 'fungi', + 'Algae': 'algae', + 'Microbes': 'microbes', + 'Biological field sampling': 'biological_field_sampling', + 'Data mining': 'data_mining', + 'Remote sensing': 'remote_sensing', + 'GIS': 'gis', + 'Spatial analysis': 'spatial_analysis', + 'Statistical analysis': 'statistical_analysis', + 'Modelling': 'modelling', + 'Assessment': 'assessment', + 'Meta-analysis': 'meta_analysis', + 'Synthesis': 'synthesis', + 'Qualitative social science methods (interviews, surveys)': 'qualitative_ssm', + 'Genetic analyses': 'genetic_analyses', + 'Field site': 'field_site', + 'Transect': 'transect', + 'Mountain top': 'mountain_top', + 'Mountain range': 'mountain_range', + 'Landscape': 'landscape', + 'Regional': 'regional', + 'National': 'national', + 'Global': 'global', + 'Geographic area of expertise': 'geographic_area_of_expertise', + 'ProfileOnWeb': 'profile_on_web', + 'Updated': 'updated', + 'ORCID': 'orcid', + 'WebOfScience': 'web_of_science', + 'Twitter': 'twitter', + 'Instagram': 'instagram', + 'ScientificName': 'scientific_name', + 'Class': 'class', + 'EnglishName': 'english_name', + 'Language': 'language', + 'Person': 'person', + 'Function': 'function', + 'Range': 'range', + 'Endemic': 'endemic', + 'SourceURL': 'source_url', + 'MountainRange': 'mountain_range', + 'TaxonRangeID': 'id', + 'SubRangeOrRegion': 'subrange_or_region', + 'Taxon': 'taxon', + 'Distribution': 'distribution', + 'RedList': 'red_list', + 'CountUnit': 'count_unit', + 'NumberUnits': 'number_of_units', + 'Remarks': 'remarks', + 'RangeType': 'range_type', + 'Role': 'role', + 'RedListCategory': 'red_list_category' + } + + def add_arguments(self, parser): + parser.add_argument('--path', type=str, help="file path") + parser.add_argument('--model_name', type=str, help="model name", required=True) + parser.add_argument('--app_name', type=str, help="django app name that the model is connected to", default='app', required=True) + + def handle(self, *args, **options): + file_path = options['path'] + _model = apps.get_model(options['app_name'], options['model_name']) + with open(file_path, 'r') as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='|') + first = True + for row in reader: + if first: + # Assume the first row to be the header + header = row + header = [h.strip('"') for h in header] + first = False + continue + _object_dict = {self.cols_to_django_fields.get(key): value.lstrip('"').rstrip('"') for key, value in zip(header, row)} + m = _model(**_object_dict) + m.save() \ No newline at end of file