initial push for version 0.0.2

This commit is contained in:
Dominique Roux 2020-07-16 15:30:51 +02:00
commit 10a617aea9
8 changed files with 404 additions and 0 deletions

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
*.pyc
__pycache__
flashairup/flashairup.conf
flashairup/data/*

65
README.md Normal file
View File

@ -0,0 +1,65 @@
# flashair uploader
## Installation requirements
python3, python3-setuptools
`sudo apt install python3-setuptools`
libcurl4-openssl-dev libssl-dev
`sudo apt install libcurl4-openssl-dev libssl-dev`
Move the `flashairup` directory to the desired place (e.g. `/home/<USER>/`), referred now as WORKDIR
Create the following directory structure inside the WORKDIR:
```
├── flashairup
│   ├── data
│   │   ├── cam1
│   │   │   ├── last.txt
│   │   │   └── pictures
│   │   └── ...
│   ├── flashairup.conf
│   └── ...
└── ...
```
where `cam1` is only an example for the name of a cam.
The `last.txt` is needed for keeping track of the index of the last synchronised pictures.
The `flashairup.conf` should at least contain the `DEFAULT` section:
```
[DEFAULT]
ftp_srv = FTP-SERVER
ftp_path = PATH
ftp_user = USER
ftp_pwd = PASSWORD
db = new
# Set the logger level (DEBUG, INFO, WARNING, ERROR)
log_level = DEBUG
```
Add a cam section to the config file:
```
[cam1] # cam name
ip = CAM-IP
location = LOCATION-OF-THE-PICTURES-ON-CAM
```
Example config:
```
[DEFAULT]
ftp_srv = my.ftp.server.org
ftp_path = media/pictures
ftp_user = ftpuser
ftp_pwd = strongpassword1
db = new
# Set the logger level (DEBUG, INFO, WARNING, ERROR)
log_level = DEBUG
[cam1]
ip = 10.0.0.10
location = DCIM/101NIKON
```

0
flashairup/__init__.py Normal file
View File

216
flashairup/cam.py Normal file
View File

@ -0,0 +1,216 @@
import pycurl
import os
from io import BytesIO
import subprocess
from ftplib import FTP
import signal
class Cam(object):
""" Cam object
Attributes
----------
log : logging_object
name : str
name of the cam, set by the section in the config file
ip : str
IP of the cam in string format
location : str
Location of the picutres on the SD-Card
ftp_config : config_object
FTP configuration; such as URI, username and password
Methods
-------
getFileList()
Get the file list of the cam via curl
organiseMissing(files)
Get a list of files and decide which to download
getImage(filename)
Get Images from cam via curl
uploadImage(filename)
Upload the image to remote FTP server
isOnline()
Check if the cam is online via ping
"""
def __init__(self, log, name, ip, location, ftp_config):
self.log = log
self.name = name
self.ip = ip
self.getFileListPath = 'http://' + self.ip + '/command.cgi?op=100&DIR=' + location
self.getFilePath = 'http://' + self.ip + '/' + location
self.ftp_config = ftp_config
#TODO: Test what happens if file is not present
fname = './data/' + self.name + '/last.txt'
self.last = str(0)
try:
with open(fname) as last_file:
for line in last_file:
self.last = line.strip().split()[0]
except FileNotFoundError:
self.log.debug(self.name + " last.txt was not found, default value 0 is set")
self.log.debug(self.name + " last index is " + self.last)
def getFileList(self):
"""Get the file list of the cam via curl
Return: list of filenames
"""
# Assert that the cam is online, otherwise the whole function doesn't make sense
assert self.isOnline()
self.log.info(self.name + " get list of files")
# Open, run and close the curl session
crl = pycurl.Curl()
b_obj = BytesIO()
crl.setopt(pycurl.CONNECTTIMEOUT, 5)
crl.setopt(pycurl.TIMEOUT, 5)
crl.setopt(crl.URL, self.getFileListPath)
crl.setopt(crl.WRITEDATA, b_obj)
crl.perform()
crl.close()
# Get the file list
get_list = b_obj.getvalue().decode('utf8').splitlines()
del get_list[0]
# Create empty list for processing later
files = []
# Get the filenames only
for line in get_list:
files.append(line.split(',')[1])
self.log.debug(self.name + " " + " ".join(files))
# Return the file list
return files
def organiseMissing(self, files):
"""Get a list of files and decide which to download
Parameters
----------
files: list
List of files
"""
# Go trough the filenames, assuming they are sorted
#TODO: If possible try to make it work without the assumption
# Do this with another index (current highest) and update this index only with a bigger number
# In the last step, update self.last with current highest index (and obviously update the value in the file
for filename in files:
file_index = int(''.join(list(filter(str.isdigit, filename))))
# Check if the file needs to be downloaded
if file_index > int(self.last):
self.log.info("Found a file: "+ filename)
# Download the file
self.getImage(filename)
self.log.info("Image " + filename + " downloaded")
try:
# Upload images to corresponding FTP server
self.uploadImage(filename)
self.log.info("File " + filename + " uploaded to remote server")
# Remove the local file
os.remove('./data/' + self.name + '/pictures/' + filename)
self.log.info("File " + filename + " removed from local storage")
# Update the last index
self.last = file_index
except ConnectionRefusedError:
self.log.error("Could not reach FTP server, exit now")
return 1
# Update the last file for each cam only once
last_file = open('./data/' + self.name + '/last.txt', "w+")
last_file.write(str(self.last))
last_file.close()
return 0
def getImage(self, filename):
"""Get Images from cam via curl
Parameters
----------
filename : str
Filnema of the image to download
"""
assert self.isOnline()
# Initialize the curl object, set timeout to 5 seconds, otherwise curl might hang if
# the cam is suddenly not reachable anymore
crl = pycurl.Curl()
crl.setopt(pycurl.CONNECTTIMEOUT, 5)
crl.setopt(pycurl.TIMEOUT, 5)
self.log.info("Download Picture from " + self.getFilePath + '/' + filename)
#TODO: Set data path in default config section
with open('./data/'+ self.name +'/pictures/' + filename, 'wb') as f:
signal.signal(signal.SIGALRM, self.handler)
signal.alarm(5)
crl.setopt(crl.URL, self.getFilePath + '/' + filename)
crl.setopt(crl.WRITEDATA, f)
crl.perform()
crl.close()
signal.alarm(0)
def uploadImage(self, filename):
"""Upload the image to remote FTP server
Parameters
----------
filename : str
Filename for the file which should be uploaded
"""
# Only if the config is set to new use the additional prefix, otherwise none
if self.ftp_config['db'] == "new":
prefix = "_neue_db"
else:
prefix = ""
# Initialize FTP settings
ftp_path = self.ftp_config['ftp_path'] + '/' + self.name + prefix
#TODO: Move to config file
ftp = FTP("xray876.server4you.net")
ftp.login(self.ftp_config['ftp_user'], self.ftp_config['ftp_pwd'])
ftp.cwd(ftp_path)
with open('data/' + self.name + '/pictures/' + filename, 'rb') as f:
ftp.storbinary('STOR %s' % os.path.basename(filename), f)
ftp.quit()
def isOnline(self):
"""Check if the Cam is online via ping
Returns
-------
0
cam is online
1
cam is offline
"""
command = ['ping', '-c', '1', '-w', '1', '-q', self.ip]
return subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
def handler(self, signum, frame):
self.log.debug("Signal handler called with signal" + signum)
raise OSError("Could not reach camera")

33
flashairup/config.py Normal file
View File

@ -0,0 +1,33 @@
import configparser
class Config(object):
""" Config Class, parses the arguments and the config from the flashairup.conf ini file"""
def __init__(self, arguments):
""" read arguments dicts as a base """
self.arguments = arguments
# Read settings from flashairup.conf ini configfile
self.configfile = "flashairup.conf"
self.conf = configparser.ConfigParser()
def load(self):
""" loads the config file"""
self.conf.read(self.configfile)
# def write(self):
# print("writing the following config to file..\n")
# self.print()
# with open('flashairup.conf', 'w') as configf:
# self.conf.write(configf)
#
#
# def print(self):
# for section in ['DEFAULT'] + self.conf.sections() :
# print('[' + section + ']')
# for key in self.conf[section]:
# print(key + ' = ' + self.conf[section][key])
# print('\n')

82
flashairup/main.py Executable file
View File

@ -0,0 +1,82 @@
#!/usr/bin/python
import argparse
import logging
import logging.handlers
from config import Config
from cam import Cam
### Arguments
arg_parser = argparse.ArgumentParser(description='flashair uploader')
# check if cams are online, if yes download all new pictures and upload them to the remote server
arg_parser.add_argument('--sync', help="synchronise pictures from the cams to the ftp server", action='store_true')
arguments = vars(arg_parser.parse_args())
### logger
FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT)
log = logging.getLogger(__name__)
# Also log output to syslog
handler = logging.handlers.SysLogHandler(address = '/dev/log')
fmt = 'flashair-uploader[%(process)-5s:%(thread)d]: ' \
'%(levelname)-5s %(message)s'
handler.setFormatter(logging.Formatter(fmt=fmt))
log.addHandler(handler)
### Main function
def main(arguments):
"""
Gets the parsed arguments and loads the configurations.
If the arguments was "--sync" the sync procedure starts
Input: parsed arguments
Output: NULL
"""
# load the configuration
log.debug("loading configuration")
config = Config(arguments)
config.load()
log.info("configuration loaded")
# Set the defined log level
try:
log.setLevel(config.conf['DEFAULT']['log_level'])
except ValueError:
log.error("The configured LogLevel is wrong " + config.conf['DEFAULT']['log_level'])
log.debug("LogLevel: " + config.conf['DEFAULT']['log_level'])
# Do a sync if correct arguments are passed
if arguments['sync']:
log.debug("syncing pictures from cams")
sync(config)
def sync(config):
"""
Starts the sync procedure for all cameras if and only if they are online
Input: config object
Output: NULL
"""
# Loads all cams
cams = config.conf.sections()
# Iterate through the cams
for cam_i in cams:
log.debug("initialize cam object for " + cam_i)
# Creates a Cam object for each cam
cam = Cam(log, cam_i, config.conf[cam_i]['ip'], config.conf[cam_i]['location'], config.conf.defaults())
try:
files = cam.getFileList()
cam.organiseMissing(files)
except AssertionError:
log.error(cam_i + " at IP: " + cam.ip + " is offline")
# Start with the main function
main(arguments)

1
flashairup/version.py Normal file
View File

@ -0,0 +1 @@
VERSION = "0.0.2"

3
requirements.txt Normal file
View File

@ -0,0 +1,3 @@
config==0.5.0.post0
configparser==5.0.0
pycurl==7.43.0.5