Add basic csv importer
To test: ./manage.py import --path /home/pcoder/Downloads/gmbadb/csvs/v2-LU_RedListCategories.csv --model_name RedListCategory --app_name app
This commit is contained in:
PCoder
parent
44fb9c3f20
commit
ec5f4111ac
|
|
@ -0,0 +1,291 @@
|
|||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.apps import apps
|
||||
|
||||
from app.models import Country
|
||||
import csv
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Imports csv to DB'
|
||||
csv_files = [
|
||||
"v2-LU_GMBA_SpeciesGroups.csv",
|
||||
"v2-LU_Countries.csv",
|
||||
"v2-LU_Languages.csv",
|
||||
"v2-LU_Sources.csv",
|
||||
"v2-LU_RedListCategories.csv",
|
||||
"v2-LU_RangeTypes.csv",
|
||||
"v2-LU_PeopleStatus.csv",
|
||||
"v2-LU_TrendsQuantity.csv",
|
||||
"v2-LU_TrendsQuality.csv",
|
||||
"v2-LU_TaxonUnit.csv",
|
||||
"v2-LU_TaxonStatus.csv",
|
||||
"v2-AddElevations.csv",
|
||||
"v2-GMBA_Function.csv",
|
||||
"v2-Gmba_V2_centroid.csv",
|
||||
"v2-ImportGeom210915.csv",
|
||||
"v2-LanguageLink.csv",
|
||||
"v2-Keywords.csv",
|
||||
"v2-NamesImport.csv",
|
||||
"v2-Organisations.csv",
|
||||
"v2-Peaks.csv",
|
||||
"v2-PeopleRanges.csv",
|
||||
"v2-PeopleFunction.csv",
|
||||
"v2-PeopleResources.csv",
|
||||
"v2-RangeCountries.csv",
|
||||
"v2-RangeNameTranslations.csv",
|
||||
"v2-RangeOnlineInfo.csv",
|
||||
"v2-Ranges.csv",
|
||||
"v2-ResourceRanges.csv",
|
||||
"v2-ResourceKeywords.csv",
|
||||
"v2-Repositories.csv",
|
||||
"v2-Resources.csv",
|
||||
"v2-Species.csv",
|
||||
"v2-Searches.csv",
|
||||
"v2-TaxonRange.csv",
|
||||
"v2-SpeciesRange.csv",
|
||||
"v2-People.csv"
|
||||
]
|
||||
cols_to_django_fields = {
|
||||
"ID": 'id',
|
||||
"Source": 'source',
|
||||
"RangeName": 'range_name',
|
||||
"LanguageTranslation": 'language_translation',
|
||||
"RangeNameTranslation": 'range_name_translation',
|
||||
"GMBA_ID_v2": 'gmba_v2_id',
|
||||
"Elev_Min": 'elev_min',
|
||||
"Elev_Max": 'elev_max',
|
||||
"Elev_Range": 'elev_range',
|
||||
"TaxonStatus": 'taxon_status',
|
||||
"InfoSource": 'info_source',
|
||||
"URL": 'url',
|
||||
"GMBA function": 'GMBA_function',
|
||||
"TaxonUnit": 'taxon_unit',
|
||||
"Range_ID": 'id',
|
||||
"RangeNameMap": 'range_name_map',
|
||||
"RangeNameAscii": 'range_name_ascii',
|
||||
"RangeNameLanguage": 'range_name_language',
|
||||
"MotherRange": 'mother_range',
|
||||
"Feature": 'feature',
|
||||
"MapUnit": 'map_unit',
|
||||
"Level": 'level',
|
||||
"LevelText": 'level_text',
|
||||
"Level_1": 'level_1',
|
||||
"Level_2": 'level_2',
|
||||
"Level_3": 'level_3',
|
||||
"Latitude": 'latitude',
|
||||
"Longitude": 'longitude',
|
||||
"Orogeny": 'orogeny',
|
||||
"Area": 'area',
|
||||
"GMBA_V1_ID": 'GMBA_v1_id',
|
||||
"Countries": 'countries',
|
||||
"Peak_Elevation": 'peak_elevation',
|
||||
"Peak_Name": 'peak_name',
|
||||
"Peak_Latitude": 'peak_latitude',
|
||||
"Peak_Longitude": 'peak_longitude',
|
||||
"Comments": 'comments',
|
||||
"Checked": 'checked',
|
||||
"Range_AlternateID": 'range_alternate_id',
|
||||
"GeologicRegion": 'geologic_region',
|
||||
"GMBA_V2_ID": 'gmba_v2_id',
|
||||
"GMBA_V2_ID_str": 'gmba_v2_id_str',
|
||||
"WikiDataID": 'wiki_data_id',
|
||||
"WikiDataURL": 'wiki_data_url',
|
||||
"Select_300": 'select_300',
|
||||
"Gmba_Narrow": 'gmba_narrow',
|
||||
"Name_FR": 'name_fr',
|
||||
"Name_DE": 'name_de',
|
||||
"Name_ES": 'name_es',
|
||||
"Name_PT": 'name_pt',
|
||||
"Name_CN": 'name_cn',
|
||||
"Name_RU": 'name_ru',
|
||||
"Name_TR": 'name_tr',
|
||||
"Perimeter": 'perimeter',
|
||||
"ColorAll": 'color_all',
|
||||
"ColorBasic": 'color_basic',
|
||||
"Color300": 'color_300',
|
||||
"Elev_Low": 'elev_low',
|
||||
"Elev_High": 'elev_high',
|
||||
"Elev_Avg": 'elev_avg',
|
||||
"gridcode": 'gridcode',
|
||||
"Trend": 'trend',
|
||||
"RepositoryName": 'repository_name',
|
||||
"RepositoryURL": 'repository_url',
|
||||
"Resource": 'resource',
|
||||
"Keyword": 'keyword',
|
||||
"Keyword_ID": 'keyword_id',
|
||||
"Mother": 'mother',
|
||||
"CN": 'cn',
|
||||
"DE": 'de',
|
||||
"ES": 'es',
|
||||
"FR": 'fr',
|
||||
"PT": 'pt',
|
||||
"RU": 'ru',
|
||||
"TR": 'tr',
|
||||
"ResourceTitle": 'resource_title',
|
||||
"LanguageLetterCode": 'language_letter_code',
|
||||
"LanguageNumberCode": 'language_number_code',
|
||||
"OrgNum1": 'org_num1',
|
||||
"Organisation Search": 'organisation_search',
|
||||
"OrgAlphaSearch": 'org_alpha_search',
|
||||
"Organisation English": 'organisation_english',
|
||||
"Organisation 2": 'organisation_2',
|
||||
"Organisation 3": 'organisation_3',
|
||||
"Organisation Original": 'organisation_original',
|
||||
"Acronym": 'acronym',
|
||||
"Street": 'street',
|
||||
"PO Box": 'po_box',
|
||||
"Postcode": 'postcode',
|
||||
"City": 'city',
|
||||
"Region": 'region',
|
||||
"SearchURL": 'search_url',
|
||||
"LatLon": 'lat_lon',
|
||||
"URL Org": 'url',
|
||||
"Tel Org": 'tel',
|
||||
"Email Org": 'email',
|
||||
"Country": 'country',
|
||||
"Tags": 'tags',
|
||||
"Description": 'description',
|
||||
"Northing": 'northing',
|
||||
"Easting": 'easting',
|
||||
"Category": 'category',
|
||||
"Subject": 'subject',
|
||||
"Title": 'title',
|
||||
"Citation": 'citation',
|
||||
"Type": 'type',
|
||||
"Abstract": 'abstract',
|
||||
"AuthorKeywords": 'author_keywords',
|
||||
"Lat": 'lat',
|
||||
"Lon": 'lon',
|
||||
"Stars": 'stars',
|
||||
"PEGASuS_Check_map_with_author": 'PEGASuS_Check_map_with_author',
|
||||
"PEGASuS_polygon_ID": 'PEGASuS_polygon_ID',
|
||||
"PEGASuS_Polygon_comments": 'PEGASuS_Polygon_comments',
|
||||
"PEGASuS_Assessment_ID": 'PEGASuS_Assessment_ID',
|
||||
"GLORIA": 'gloria',
|
||||
"GNOMO": 'gnomo',
|
||||
"LTER": 'lter',
|
||||
"LTSER": 'ltser',
|
||||
"MIREN": 'miren',
|
||||
"TEAM": 'team',
|
||||
"Inventory": 'inventory',
|
||||
"DOI": 'doi',
|
||||
"ShortName": 'short_name',
|
||||
"FormalName": 'formal_name',
|
||||
"Membership within the UN System": '',
|
||||
"Continent": 'continent',
|
||||
"EU_MS": 'eu_ms',
|
||||
"EEA_MS": 'eea_ms',
|
||||
"ISO3": 'iso3',
|
||||
"ISO2": 'iso2',
|
||||
"Point_Name": 'point_name',
|
||||
"Elevation": 'elevation',
|
||||
"Link": 'link',
|
||||
"Repository": 'repository',
|
||||
"SearchString": 'search_string',
|
||||
"SearchDate": 'search_date',
|
||||
"Result": 'result',
|
||||
"NumberOfRecords": 'number_of_records',
|
||||
"Stored": 'stored',
|
||||
"SpeciesGroup": 'species_group',
|
||||
"MrMrs": 'mr_mrs',
|
||||
"First name": 'first_name',
|
||||
"Last name": 'last_name',
|
||||
"Full name": 'full_name',
|
||||
"SearchName": 'search_name',
|
||||
"e-mail 1": 'email_1',
|
||||
"e-mail 2": 'email_2',
|
||||
"Skype": 'skype',
|
||||
"Professional phone": 'professional_phone',
|
||||
"Mobile number": 'mobile_number',
|
||||
"Field of expertise": 'field_of_expertise',
|
||||
"Biography": 'biography',
|
||||
"Position": 'position',
|
||||
"Status": 'status',
|
||||
"Entry date": 'entry_date',
|
||||
"Newsletter": 'news_letter',
|
||||
"CountryLookup": 'country_lookup',
|
||||
"Organisation": 'organization',
|
||||
"Birds": 'birds',
|
||||
"Mammals": 'mammals',
|
||||
'Reptiles': 'reptiles',
|
||||
'Amphibians': 'amphibians',
|
||||
'Fish': 'fish',
|
||||
'Insects': 'insects',
|
||||
'Molluscs': 'molluscs',
|
||||
'Crustaceans': 'crustaceans',
|
||||
'Arachnids': 'arachnids',
|
||||
'Angiosperms': 'angiosperms',
|
||||
'Gymnosperms': 'gymnosperms',
|
||||
'Fungi': 'fungi',
|
||||
'Algae': 'algae',
|
||||
'Microbes': 'microbes',
|
||||
'Biological field sampling': 'biological_field_sampling',
|
||||
'Data mining': 'data_mining',
|
||||
'Remote sensing': 'remote_sensing',
|
||||
'GIS': 'gis',
|
||||
'Spatial analysis': 'spatial_analysis',
|
||||
'Statistical analysis': 'statistical_analysis',
|
||||
'Modelling': 'modelling',
|
||||
'Assessment': 'assessment',
|
||||
'Meta-analysis': 'meta_analysis',
|
||||
'Synthesis': 'synthesis',
|
||||
'Qualitative social science methods (interviews, surveys)': 'qualitative_ssm',
|
||||
'Genetic analyses': 'genetic_analyses',
|
||||
'Field site': 'field_site',
|
||||
'Transect': 'transect',
|
||||
'Mountain top': 'mountain_top',
|
||||
'Mountain range': 'mountain_range',
|
||||
'Landscape': 'landscape',
|
||||
'Regional': 'regional',
|
||||
'National': 'national',
|
||||
'Global': 'global',
|
||||
'Geographic area of expertise': 'geographic_area_of_expertise',
|
||||
'ProfileOnWeb': 'profile_on_web',
|
||||
'Updated': 'updated',
|
||||
'ORCID': 'orcid',
|
||||
'WebOfScience': 'web_of_science',
|
||||
'Twitter': 'twitter',
|
||||
'Instagram': 'instagram',
|
||||
'ScientificName': 'scientific_name',
|
||||
'Class': 'class',
|
||||
'EnglishName': 'english_name',
|
||||
'Language': 'language',
|
||||
'Person': 'person',
|
||||
'Function': 'function',
|
||||
'Range': 'range',
|
||||
'Endemic': 'endemic',
|
||||
'SourceURL': 'source_url',
|
||||
'MountainRange': 'mountain_range',
|
||||
'TaxonRangeID': 'id',
|
||||
'SubRangeOrRegion': 'subrange_or_region',
|
||||
'Taxon': 'taxon',
|
||||
'Distribution': 'distribution',
|
||||
'RedList': 'red_list',
|
||||
'CountUnit': 'count_unit',
|
||||
'NumberUnits': 'number_of_units',
|
||||
'Remarks': 'remarks',
|
||||
'RangeType': 'range_type',
|
||||
'Role': 'role',
|
||||
'RedListCategory': 'red_list_category'
|
||||
}
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--path', type=str, help="file path")
|
||||
parser.add_argument('--model_name', type=str, help="model name", required=True)
|
||||
parser.add_argument('--app_name', type=str, help="django app name that the model is connected to", default='app', required=True)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
file_path = options['path']
|
||||
_model = apps.get_model(options['app_name'], options['model_name'])
|
||||
with open(file_path, 'r') as csv_file:
|
||||
reader = csv.reader(csv_file, delimiter=',', quotechar='|')
|
||||
first = True
|
||||
for row in reader:
|
||||
if first:
|
||||
# Assume the first row to be the header
|
||||
header = row
|
||||
header = [h.strip('"') for h in header]
|
||||
first = False
|
||||
continue
|
||||
_object_dict = {self.cols_to_django_fields.get(key): value.lstrip('"').rstrip('"') for key, value in zip(header, row)}
|
||||
m = _model(**_object_dict)
|
||||
m.save()
|
||||
Loading…
Reference in New Issue