From 10a617aea9c1c8f286f1a5722c3fbc15f31d00d5 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Thu, 16 Jul 2020 15:30:51 +0200 Subject: [PATCH] initial push for version 0.0.2 --- .gitignore | 4 + README.md | 65 +++++++++++++ flashairup/__init__.py | 0 flashairup/cam.py | 216 +++++++++++++++++++++++++++++++++++++++++ flashairup/config.py | 33 +++++++ flashairup/main.py | 82 ++++++++++++++++ flashairup/version.py | 1 + requirements.txt | 3 + 8 files changed, 404 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 flashairup/__init__.py create mode 100644 flashairup/cam.py create mode 100644 flashairup/config.py create mode 100755 flashairup/main.py create mode 100644 flashairup/version.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e54cde3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.pyc +__pycache__ +flashairup/flashairup.conf +flashairup/data/* diff --git a/README.md b/README.md new file mode 100644 index 0000000..d48b909 --- /dev/null +++ b/README.md @@ -0,0 +1,65 @@ +# flashair uploader + + +## Installation requirements + +python3, python3-setuptools +`sudo apt install python3-setuptools` +libcurl4-openssl-dev libssl-dev +`sudo apt install libcurl4-openssl-dev libssl-dev` + + +Move the `flashairup` directory to the desired place (e.g. `/home//`), referred now as WORKDIR + +Create the following directory structure inside the WORKDIR: + +``` +├── flashairup +│   ├── data +│   │   ├── cam1 +│   │   │   ├── last.txt +│   │   │   └── pictures +│   │   └── ... +│   ├── flashairup.conf +│   └── ... +└── ... +``` +where `cam1` is only an example for the name of a cam. +The `last.txt` is needed for keeping track of the index of the last synchronised pictures. +The `flashairup.conf` should at least contain the `DEFAULT` section: + +``` + [DEFAULT] + ftp_srv = FTP-SERVER + ftp_path = PATH + ftp_user = USER + ftp_pwd = PASSWORD + db = new + # Set the logger level (DEBUG, INFO, WARNING, ERROR) + log_level = DEBUG +``` + +Add a cam section to the config file: + +``` + [cam1] # cam name + ip = CAM-IP + location = LOCATION-OF-THE-PICTURES-ON-CAM +``` + +Example config: + +``` + [DEFAULT] + ftp_srv = my.ftp.server.org + ftp_path = media/pictures + ftp_user = ftpuser + ftp_pwd = strongpassword1 + db = new + # Set the logger level (DEBUG, INFO, WARNING, ERROR) + log_level = DEBUG + + [cam1] + ip = 10.0.0.10 + location = DCIM/101NIKON +``` diff --git a/flashairup/__init__.py b/flashairup/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flashairup/cam.py b/flashairup/cam.py new file mode 100644 index 0000000..d172e22 --- /dev/null +++ b/flashairup/cam.py @@ -0,0 +1,216 @@ +import pycurl +import os +from io import BytesIO +import subprocess +from ftplib import FTP +import signal + +class Cam(object): + """ Cam object + + Attributes + ---------- + log : logging_object + name : str + name of the cam, set by the section in the config file + ip : str + IP of the cam in string format + location : str + Location of the picutres on the SD-Card + ftp_config : config_object + FTP configuration; such as URI, username and password + + + Methods + ------- + getFileList() + Get the file list of the cam via curl + organiseMissing(files) + Get a list of files and decide which to download + getImage(filename) + Get Images from cam via curl + uploadImage(filename) + Upload the image to remote FTP server + isOnline() + Check if the cam is online via ping + + """ + def __init__(self, log, name, ip, location, ftp_config): + self.log = log + self.name = name + self.ip = ip + self.getFileListPath = 'http://' + self.ip + '/command.cgi?op=100&DIR=' + location + self.getFilePath = 'http://' + self.ip + '/' + location + self.ftp_config = ftp_config + + #TODO: Test what happens if file is not present + fname = './data/' + self.name + '/last.txt' + self.last = str(0) + try: + with open(fname) as last_file: + for line in last_file: + self.last = line.strip().split()[0] + except FileNotFoundError: + self.log.debug(self.name + " last.txt was not found, default value 0 is set") + + self.log.debug(self.name + " last index is " + self.last) + + + def getFileList(self): + """Get the file list of the cam via curl + + Return: list of filenames + """ + + # Assert that the cam is online, otherwise the whole function doesn't make sense + assert self.isOnline() + + self.log.info(self.name + " get list of files") + + # Open, run and close the curl session + crl = pycurl.Curl() + b_obj = BytesIO() + crl.setopt(pycurl.CONNECTTIMEOUT, 5) + crl.setopt(pycurl.TIMEOUT, 5) + crl.setopt(crl.URL, self.getFileListPath) + crl.setopt(crl.WRITEDATA, b_obj) + crl.perform() + crl.close() + + # Get the file list + get_list = b_obj.getvalue().decode('utf8').splitlines() + del get_list[0] + + # Create empty list for processing later + files = [] + + # Get the filenames only + for line in get_list: + files.append(line.split(',')[1]) + + self.log.debug(self.name + " " + " ".join(files)) + + # Return the file list + return files + + def organiseMissing(self, files): + """Get a list of files and decide which to download + + Parameters + ---------- + files: list + List of files + """ + # Go trough the filenames, assuming they are sorted + #TODO: If possible try to make it work without the assumption + # Do this with another index (current highest) and update this index only with a bigger number + # In the last step, update self.last with current highest index (and obviously update the value in the file + for filename in files: + file_index = int(''.join(list(filter(str.isdigit, filename)))) + + # Check if the file needs to be downloaded + if file_index > int(self.last): + self.log.info("Found a file: "+ filename) + # Download the file + self.getImage(filename) + self.log.info("Image " + filename + " downloaded") + + try: + # Upload images to corresponding FTP server + self.uploadImage(filename) + self.log.info("File " + filename + " uploaded to remote server") + # Remove the local file + os.remove('./data/' + self.name + '/pictures/' + filename) + self.log.info("File " + filename + " removed from local storage") + # Update the last index + self.last = file_index + except ConnectionRefusedError: + self.log.error("Could not reach FTP server, exit now") + return 1 + + + + # Update the last file for each cam only once + last_file = open('./data/' + self.name + '/last.txt', "w+") + last_file.write(str(self.last)) + last_file.close() + + return 0 + + + def getImage(self, filename): + """Get Images from cam via curl + + Parameters + ---------- + filename : str + Filnema of the image to download + """ + + assert self.isOnline() + + # Initialize the curl object, set timeout to 5 seconds, otherwise curl might hang if + # the cam is suddenly not reachable anymore + crl = pycurl.Curl() + crl.setopt(pycurl.CONNECTTIMEOUT, 5) + crl.setopt(pycurl.TIMEOUT, 5) + self.log.info("Download Picture from " + self.getFilePath + '/' + filename) + + #TODO: Set data path in default config section + with open('./data/'+ self.name +'/pictures/' + filename, 'wb') as f: + signal.signal(signal.SIGALRM, self.handler) + signal.alarm(5) + crl.setopt(crl.URL, self.getFilePath + '/' + filename) + crl.setopt(crl.WRITEDATA, f) + crl.perform() + crl.close() + signal.alarm(0) + + + def uploadImage(self, filename): + """Upload the image to remote FTP server + + Parameters + ---------- + filename : str + Filename for the file which should be uploaded + + """ + # Only if the config is set to new use the additional prefix, otherwise none + if self.ftp_config['db'] == "new": + prefix = "_neue_db" + else: + prefix = "" + + # Initialize FTP settings + ftp_path = self.ftp_config['ftp_path'] + '/' + self.name + prefix + #TODO: Move to config file + ftp = FTP("xray876.server4you.net") + ftp.login(self.ftp_config['ftp_user'], self.ftp_config['ftp_pwd']) + ftp.cwd(ftp_path) + + with open('data/' + self.name + '/pictures/' + filename, 'rb') as f: + ftp.storbinary('STOR %s' % os.path.basename(filename), f) + + ftp.quit() + + + def isOnline(self): + """Check if the Cam is online via ping + + Returns + ------- + 0 + cam is online + 1 + cam is offline + """ + command = ['ping', '-c', '1', '-w', '1', '-q', self.ip] + + return subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 + + def handler(self, signum, frame): + self.log.debug("Signal handler called with signal" + signum) + raise OSError("Could not reach camera") + + diff --git a/flashairup/config.py b/flashairup/config.py new file mode 100644 index 0000000..96882ee --- /dev/null +++ b/flashairup/config.py @@ -0,0 +1,33 @@ +import configparser + + +class Config(object): + """ Config Class, parses the arguments and the config from the flashairup.conf ini file""" + def __init__(self, arguments): + """ read arguments dicts as a base """ + + self.arguments = arguments + + # Read settings from flashairup.conf ini configfile + self.configfile = "flashairup.conf" + self.conf = configparser.ConfigParser() + + + def load(self): + """ loads the config file""" + self.conf.read(self.configfile) + + +# def write(self): +# print("writing the following config to file..\n") +# self.print() +# with open('flashairup.conf', 'w') as configf: +# self.conf.write(configf) +# +# +# def print(self): +# for section in ['DEFAULT'] + self.conf.sections() : +# print('[' + section + ']') +# for key in self.conf[section]: +# print(key + ' = ' + self.conf[section][key]) +# print('\n') diff --git a/flashairup/main.py b/flashairup/main.py new file mode 100755 index 0000000..ed3580a --- /dev/null +++ b/flashairup/main.py @@ -0,0 +1,82 @@ +#!/usr/bin/python +import argparse +import logging +import logging.handlers + +from config import Config +from cam import Cam + + +### Arguments +arg_parser = argparse.ArgumentParser(description='flashair uploader') +# check if cams are online, if yes download all new pictures and upload them to the remote server +arg_parser.add_argument('--sync', help="synchronise pictures from the cams to the ftp server", action='store_true') + +arguments = vars(arg_parser.parse_args()) + + +### logger +FORMAT = '%(asctime)-15s %(message)s' +logging.basicConfig(format=FORMAT) +log = logging.getLogger(__name__) +# Also log output to syslog +handler = logging.handlers.SysLogHandler(address = '/dev/log') +fmt = 'flashair-uploader[%(process)-5s:%(thread)d]: ' \ + '%(levelname)-5s %(message)s' +handler.setFormatter(logging.Formatter(fmt=fmt)) +log.addHandler(handler) + +### Main function +def main(arguments): + """ + Gets the parsed arguments and loads the configurations. + If the arguments was "--sync" the sync procedure starts + + Input: parsed arguments + Output: NULL + """ + # load the configuration + log.debug("loading configuration") + config = Config(arguments) + config.load() + log.info("configuration loaded") + + # Set the defined log level + try: + log.setLevel(config.conf['DEFAULT']['log_level']) + except ValueError: + log.error("The configured LogLevel is wrong " + config.conf['DEFAULT']['log_level']) + + log.debug("LogLevel: " + config.conf['DEFAULT']['log_level']) + + # Do a sync if correct arguments are passed + if arguments['sync']: + log.debug("syncing pictures from cams") + sync(config) + + + +def sync(config): + """ + Starts the sync procedure for all cameras if and only if they are online + + Input: config object + Output: NULL + """ + # Loads all cams + cams = config.conf.sections() + # Iterate through the cams + for cam_i in cams: + log.debug("initialize cam object for " + cam_i) + # Creates a Cam object for each cam + cam = Cam(log, cam_i, config.conf[cam_i]['ip'], config.conf[cam_i]['location'], config.conf.defaults()) + try: + files = cam.getFileList() + cam.organiseMissing(files) + except AssertionError: + log.error(cam_i + " at IP: " + cam.ip + " is offline") + + + +# Start with the main function +main(arguments) diff --git a/flashairup/version.py b/flashairup/version.py new file mode 100644 index 0000000..bc4ffb3 --- /dev/null +++ b/flashairup/version.py @@ -0,0 +1 @@ +VERSION = "0.0.2" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..93d3303 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +config==0.5.0.post0 +configparser==5.0.0 +pycurl==7.43.0.5