initial push for version 0.0.2

.gitignore

@@ -0,0 +1,4 @@
+*.pyc
+__pycache__
+flashairup/flashairup.conf
+flashairup/data/*

README.md

@@ -0,0 +1,65 @@
+# flashair uploader
+
+
+## Installation requirements
+
+python3, python3-setuptools
+`sudo apt install python3-setuptools`
+libcurl4-openssl-dev libssl-dev
+`sudo apt install libcurl4-openssl-dev libssl-dev`
+
+
+Move the `flashairup` directory to the desired place (e.g. `/home/<USER>/`), referred now as WORKDIR
+
+Create the following directory structure inside the WORKDIR:
+
+```
+├── flashairup
+│   ├── data
+│   │   ├── cam1
+│   │   │   ├── last.txt
+│   │   │   └── pictures
+│   │   └── ...
+│   ├── flashairup.conf
+│   └── ...
+└── ...
+```
+where `cam1` is only an example for the name of a cam.
+The `last.txt` is needed for keeping track of the index of the last synchronised pictures.
+The `flashairup.conf` should at least contain the `DEFAULT` section:
+
+```
+   [DEFAULT]
+   ftp_srv = FTP-SERVER
+   ftp_path = PATH
+   ftp_user = USER
+   ftp_pwd = PASSWORD
+   db = new
+   # Set the logger level (DEBUG, INFO, WARNING, ERROR)
+   log_level = DEBUG
+```
+
+Add a cam section to the config file:
+
+```
+   [cam1] # cam name
+   ip = CAM-IP
+   location = LOCATION-OF-THE-PICTURES-ON-CAM
+```
+
+Example config:
+
+```
+   [DEFAULT]
+   ftp_srv = my.ftp.server.org
+   ftp_path = media/pictures
+   ftp_user = ftpuser
+   ftp_pwd = strongpassword1
+   db = new
+   # Set the logger level (DEBUG, INFO, WARNING, ERROR)
+   log_level = DEBUG
+
+   [cam1]
+   ip = 10.0.0.10
+   location = DCIM/101NIKON
+```

flashairup/cam.py

@@ -0,0 +1,216 @@
+import pycurl
+import os
+from io import BytesIO
+import subprocess
+from ftplib import FTP
+import signal
+
+class Cam(object):
+    """ Cam object
+
+    Attributes
+    ----------
+    log : logging_object
+    name : str
+        name of the cam, set by the section in the config file
+    ip : str
+        IP of the cam in string format
+    location : str
+        Location of the picutres on the SD-Card
+    ftp_config : config_object
+        FTP configuration; such as URI, username and password
+
+
+    Methods
+    -------
+    getFileList()
+        Get the  file list of the cam via curl
+    organiseMissing(files)
+        Get a list of files and decide which to download
+    getImage(filename)
+        Get Images from cam via curl
+    uploadImage(filename)
+        Upload the image to remote FTP server
+    isOnline()
+        Check if the cam is online via ping
+
+    """
+    def __init__(self, log, name, ip, location, ftp_config):
+        self.log = log
+        self.name = name
+        self.ip = ip
+        self.getFileListPath = 'http://' + self.ip + '/command.cgi?op=100&DIR=' + location
+        self.getFilePath = 'http://' + self.ip + '/' + location
+        self.ftp_config = ftp_config
+
+        #TODO: Test what happens if file is not present
+        fname = './data/' + self.name + '/last.txt'
+        self.last = str(0)
+        try:
+            with open(fname) as last_file:
+                for line in last_file:
+                    self.last = line.strip().split()[0]
+        except FileNotFoundError:
+            self.log.debug(self.name + " last.txt was not found, default value 0 is set")
+
+        self.log.debug(self.name + " last index is " + self.last)
+ 
+ 
+    def getFileList(self):
+        """Get the file list of the cam via curl
+
+        Return: list of filenames
+        """
+
+        # Assert that the cam is online, otherwise the whole function doesn't make sense
+        assert self.isOnline()
+
+        self.log.info(self.name + " get list of files")
+
+        # Open, run and close the curl session
+        crl = pycurl.Curl()
+        b_obj = BytesIO()
+        crl.setopt(pycurl.CONNECTTIMEOUT, 5)
+        crl.setopt(pycurl.TIMEOUT, 5)
+        crl.setopt(crl.URL, self.getFileListPath)
+        crl.setopt(crl.WRITEDATA, b_obj)
+        crl.perform()
+        crl.close()
+ 
+        # Get the file list
+        get_list = b_obj.getvalue().decode('utf8').splitlines()
+        del get_list[0]
+ 
+        # Create empty list for processing later
+        files = []
+ 
+        # Get the filenames only
+        for line in get_list:
+            files.append(line.split(',')[1])
+
+        self.log.debug(self.name + " " + " ".join(files))
+ 
+        # Return the file list
+        return files
+
+    def organiseMissing(self, files):
+        """Get a list of files and decide which to download
+        
+        Parameters
+        ----------
+        files: list
+            List of files 
+        """
+        # Go trough the filenames, assuming they are sorted
+        #TODO: If possible try to make it work without the assumption
+        # Do this with another index (current highest) and update this index only with a bigger number
+        # In the last step, update self.last with current highest index (and obviously update the value in the file
+        for filename in files:
+            file_index = int(''.join(list(filter(str.isdigit, filename))))
+
+            # Check if the file needs to be downloaded
+            if file_index > int(self.last):
+                self.log.info("Found a file: "+ filename)
+                # Download the file
+                self.getImage(filename)
+                self.log.info("Image " + filename + " downloaded")
+
+                try:
+                    # Upload images to corresponding FTP server
+                    self.uploadImage(filename)
+                    self.log.info("File " + filename + " uploaded to remote server")
+                    # Remove the local file
+                    os.remove('./data/' + self.name + '/pictures/' + filename)
+                    self.log.info("File " + filename + " removed from local storage")
+                    # Update the last index
+                    self.last = file_index
+                except ConnectionRefusedError:
+                    self.log.error("Could not reach FTP server, exit now")
+                    return 1
+
+
+
+        # Update the last file for each cam only once
+        last_file = open('./data/' + self.name + '/last.txt', "w+")
+        last_file.write(str(self.last))
+        last_file.close()
+
+        return 0
+
+ 
+    def getImage(self, filename):
+        """Get Images from cam via curl
+
+        Parameters
+        ----------
+        filename : str
+            Filnema of the image to download
+        """
+
+        assert self.isOnline()
+
+        # Initialize the curl object, set timeout to 5 seconds, otherwise curl might hang if 
+        # the cam is suddenly not reachable anymore
+        crl = pycurl.Curl()
+        crl.setopt(pycurl.CONNECTTIMEOUT, 5)
+        crl.setopt(pycurl.TIMEOUT, 5)
+        self.log.info("Download Picture from " + self.getFilePath + '/' + filename)
+
+        #TODO: Set data path in default config section
+        with open('./data/'+ self.name +'/pictures/' + filename, 'wb') as f:
+            signal.signal(signal.SIGALRM, self.handler)
+            signal.alarm(5)
+            crl.setopt(crl.URL, self.getFilePath + '/' + filename)
+            crl.setopt(crl.WRITEDATA, f)
+            crl.perform()
+            crl.close()
+            signal.alarm(0)
+
+
+    def uploadImage(self, filename):
+        """Upload the image to remote FTP server
+
+        Parameters
+        ----------
+        filename : str
+            Filename for the file which should be uploaded
+
+        """
+        # Only if the config is set to new use the additional prefix, otherwise none
+        if self.ftp_config['db'] == "new":
+            prefix = "_neue_db"
+        else:
+            prefix = ""
+
+        # Initialize FTP settings
+        ftp_path = self.ftp_config['ftp_path'] + '/' +  self.name +  prefix
+        #TODO: Move to config file
+        ftp = FTP("xray876.server4you.net")
+        ftp.login(self.ftp_config['ftp_user'], self.ftp_config['ftp_pwd'])
+        ftp.cwd(ftp_path)
+
+        with open('data/' + self.name + '/pictures/' + filename, 'rb') as f:
+            ftp.storbinary('STOR %s' % os.path.basename(filename), f)
+        
+        ftp.quit()
+
+
+    def isOnline(self):
+        """Check if the Cam is online via ping
+        
+        Returns
+        -------
+        0
+            cam is online
+        1
+            cam is offline
+        """
+        command = ['ping', '-c', '1', '-w', '1', '-q', self.ip]
+
+        return subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
+
+    def handler(self, signum, frame):
+        self.log.debug("Signal handler called with signal" + signum)
+        raise OSError("Could not reach camera")
+
+

flashairup/config.py

@@ -0,0 +1,33 @@
+import configparser
+
+
+class Config(object):
+    """ Config Class, parses the arguments and the config from the flashairup.conf ini file"""
+    def __init__(self, arguments):
+        """ read arguments dicts as a base """
+
+        self.arguments = arguments
+
+        # Read settings from flashairup.conf ini configfile 
+        self.configfile = "flashairup.conf"
+        self.conf = configparser.ConfigParser()
+
+
+    def load(self):
+        """ loads the config file"""
+        self.conf.read(self.configfile)
+
+
+#    def write(self):
+#        print("writing the following config to file..\n")
+#        self.print()
+#        with open('flashairup.conf', 'w') as configf:
+#            self.conf.write(configf)
+#
+#
+#    def print(self):
+#        for section in  ['DEFAULT'] + self.conf.sections() :
+#            print('[' + section + ']')
+#            for key in self.conf[section]:
+#                print(key + ' = ' + self.conf[section][key])
+#            print('\n')

flashairup/main.py

@@ -0,0 +1,82 @@
+#!/usr/bin/python
+import argparse
+import logging
+import logging.handlers
+
+from config import Config
+from cam import Cam
+
+
+### Arguments
+arg_parser = argparse.ArgumentParser(description='flashair uploader')
+# check if cams are online, if yes download all new pictures and upload them to the remote server
+arg_parser.add_argument('--sync', help="synchronise pictures from the cams to the ftp server", action='store_true')
+
+arguments = vars(arg_parser.parse_args())
+
+
+### logger
+FORMAT = '%(asctime)-15s %(message)s'
+logging.basicConfig(format=FORMAT)
+log = logging.getLogger(__name__)
+# Also log output to syslog
+handler = logging.handlers.SysLogHandler(address = '/dev/log')
+fmt = 'flashair-uploader[%(process)-5s:%(thread)d]: ' \
+                '%(levelname)-5s %(message)s'
+handler.setFormatter(logging.Formatter(fmt=fmt))
+log.addHandler(handler)
+
+### Main function
+def main(arguments):
+    """
+    Gets the parsed arguments and loads the configurations.
+    If the arguments was "--sync" the sync procedure starts
+
+    Input: parsed arguments
+    Output: NULL
+    """
+    # load the configuration
+    log.debug("loading configuration")
+    config = Config(arguments)
+    config.load()
+    log.info("configuration loaded")
+
+    # Set the defined log level
+    try:
+        log.setLevel(config.conf['DEFAULT']['log_level'])
+    except ValueError:
+        log.error("The configured LogLevel is wrong " + config.conf['DEFAULT']['log_level'])
+
+    log.debug("LogLevel: " + config.conf['DEFAULT']['log_level'])
+
+    # Do a sync if correct arguments are passed
+    if arguments['sync']:
+        log.debug("syncing pictures from cams")
+        sync(config)
+
+
+
+def sync(config):
+    """
+    Starts the sync procedure for all cameras if and only if they are online
+
+    Input: config object
+    Output: NULL
+    """
+    # Loads all cams
+    cams = config.conf.sections()
+    # Iterate through the cams
+    for cam_i in cams:
+        log.debug("initialize cam object for " + cam_i)
+        # Creates a Cam object for each cam
+        cam = Cam(log, cam_i, config.conf[cam_i]['ip'], config.conf[cam_i]['location'], config.conf.defaults())
+        try:
+            files = cam.getFileList()
+            cam.organiseMissing(files)
+        except AssertionError:
+            log.error(cam_i + " at IP: " + cam.ip + " is offline")
+
+
+
+# Start with the main function
+main(arguments)

flashairup/version.py

@@ -0,0 +1 @@
+VERSION = "0.0.2"

requirements.txt

@@ -0,0 +1,3 @@
+config==0.5.0.post0
+configparser==5.0.0
+pycurl==7.43.0.5