diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e2b5744 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.9-alpine + +ADD ./requirements.txt /app/requirements.txt + + +RUN set -ex \ + && apk add --no-cache --virtual .build-deps postgresql-dev build-base linux-headers jpeg-dev zlib-dev gcc musl-dev python3-dev libffi-dev openssl-dev olm-dev \ + && python -m venv /env \ + && /env/bin/pip install --upgrade pip \ + && /env/bin/pip install -U setuptools \ + && /env/bin/pip install wheel \ + && /env/bin/pip install --no-cache-dir -r /app/requirements.txt \ + && runDeps="$(scanelf --needed --nobanner --recursive /env \ + | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ + | sort -u \ + | xargs -r apk info --installed \ + | sort -u)" \ + && apk add --virtual rundeps $runDeps \ + && apk del .build-deps + +ADD ./ /app +WORKDIR /app + +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +RUN pip install noti_py + +COPY ./entrypoint.sh / +ENTRYPOINT [ "sh", "entrypoint.sh" ] + + + + + + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bf0a7c3 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +tag=latest +image=crawlerapp +build: + docker build --force-rm $(options) -t crawlerapp:latest . + +compose-start: + docker-compose up --remove-orphans $(options) + +compose-stop: + docker-compose down --remove-orphans $(options) + +compose-manage-py: + docker-compose run --rm $(options) website /env/bin/python /app/manage.py createsuperuser + +migrate: + /env/bin/python /app/manage.py migrate +start-server: + gunicorn --bind :8000 --error-logfile /var/log/gunicorn-error.log --workers 3 crawlerApp.wsgi:application \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..4afe282 --- /dev/null +++ b/config.yaml @@ -0,0 +1,8 @@ +homeserver: + base: https://ungleich.matrix.ungleich.cloud + api_base: /_matrix/client/r0 +user: + token: syt_YXNhbWloYXNzYW4_KVTnvQNvKmDKmYvYQwMF_2vLBFx +room: + id: + - '!FCoFjcLuSBdTOQzVaG:ungleich.ch' diff --git a/crawlerApp/__pycache__/settings.cpython-39.pyc b/crawlerApp/__pycache__/settings.cpython-39.pyc index 8624a2b..db7763b 100644 Binary files a/crawlerApp/__pycache__/settings.cpython-39.pyc and b/crawlerApp/__pycache__/settings.cpython-39.pyc differ diff --git a/crawlerApp/settings.py b/crawlerApp/settings.py index 556b99c..718ed5c 100644 --- a/crawlerApp/settings.py +++ b/crawlerApp/settings.py @@ -20,8 +20,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'y2zbt_#gbod3w3))59#y8*(6u7-js^!-6xvw*q=l4emn)p&-ef' -#SECRET_KEY = str(os.environ.get('SECRET_KEY',default="your_key")) +# SECRET_KEY = 'y2zbt_#gbod3w3))59#y8*(6u7-js^!-6xvw*q=l4emn)p&-ef' +SECRET_KEY = str(os.environ.get('SECRET_KEY',default="your_key")) # SECURITY WARNING: don't run with debug turned on in production! #DEBUG = True @@ -85,6 +85,35 @@ DATABASES = { } +# DBNam = os.getenv('DBNam') +POSTGRES_HOST = os.environ.get('POSTGRES_HOST',default='postgres') +POSTGRES_DB = os.environ.get('POSTGRES_DB',default='postgres') +POSTGRES_USER = os.environ.get('POSTGRES_USER',default='postgres') +POSTGRES_PASSWORD = os.environ.get('POSTGRES_PASSWORD',default='postgres') + +if DEBUG == 1: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR + 'db.sqlite3', + } + } +else: + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'HOST': POSTGRES_HOST, # + 'NAME': POSTGRES_DB, # btredb local + 'USER': POSTGRES_USER, + 'PORT': 5432, + 'PASSWORD': POSTGRES_PASSWORD, + }} + + + + + # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -122,11 +151,11 @@ USE_TZ = True # https://docs.djangoproject.com/en/3.0/howto/static-files/ STATIC_URL = '/static/' -# STATIC_ROOT = BASE_DIR + '/static/' +STATIC_ROOT = BASE_DIR + '/static/' -STATICFILES_DIRS = [ - os.path.join(BASE_DIR, "static"), -] +# STATICFILES_DIRS = [ +# os.path.join(BASE_DIR, "static"), +# ] if DEBUG: EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' diff --git a/db.sqlite3 b/db.sqlite3 index 63c6c1d..cacc16d 100644 Binary files a/db.sqlite3 and b/db.sqlite3 differ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1d0190e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,60 @@ +version: '3.7' + +x-service-volumes: &service-volumes + - ./:/app/:rw,cached + - static:/app/static +x-database-variables: &database-variables + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + + +x-app-variables: &app-variables + <<: *database-variables + POSTGRES_HOST: postgres + DEBUG : 0 + + +services: + crawlerApp: + image: crawlerapp:latest + # command: ["gunicorn", "--bind", ":8000","--error-logfile" ,"/var/log/gunicorn-error.log","--workers", "1", "crawlerApp.wsgi:application"] + # expose: + # - 8000 + # #command: /env/bin/python /app/manage.py runserver 0.0.0.0:3000 + volumes: *service-volumes + environment: *app-variables + depends_on: + - db_migrate + ports: + - "8000:8000" + + db_migrate: + image: crawlerapp:latest + command: /env/bin/python /app/manage.py migrate + volumes: *service-volumes + environment: *app-variables + depends_on: + - postgres + + nginx: + build : ./nginx + ports: + - '3000:3000' + volumes: *service-volumes + depends_on: + - crawlerApp + + + + postgres: + image: postgres + ports: + - "5432:5432" + environment: *app-variables + volumes: + - db-data:/var/lib/postgresql/data + +volumes: + db-data: + static: \ No newline at end of file diff --git a/dockerhubCrawler/__pycache__/views.cpython-39.pyc b/dockerhubCrawler/__pycache__/views.cpython-39.pyc index 513bcfd..d380ec2 100644 Binary files a/dockerhubCrawler/__pycache__/views.cpython-39.pyc and b/dockerhubCrawler/__pycache__/views.cpython-39.pyc differ diff --git a/dockerhubCrawler/migrations/0013_auto_20220220_0658.py b/dockerhubCrawler/migrations/0013_auto_20220220_0658.py new file mode 100644 index 0000000..8bfb788 --- /dev/null +++ b/dockerhubCrawler/migrations/0013_auto_20220220_0658.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0 on 2022-02-20 06:58 + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('dockerhubCrawler', '0012_auto_20220213_1751'), + ] + + operations = [ + migrations.AlterField( + model_name='dockerhubcrawler', + name='last_pushed', + field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 27, 6, 58, 29, 116932, tzinfo=utc), null=True), + ), + migrations.AlterField( + model_name='dockerhubcrawler', + name='last_updated', + field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 27, 6, 58, 29, 116976, tzinfo=utc), null=True), + ), + ] diff --git a/dockerhubCrawler/migrations/__pycache__/0013_auto_20220220_0658.cpython-39.pyc b/dockerhubCrawler/migrations/__pycache__/0013_auto_20220220_0658.cpython-39.pyc new file mode 100644 index 0000000..0bf3eed Binary files /dev/null and b/dockerhubCrawler/migrations/__pycache__/0013_auto_20220220_0658.cpython-39.pyc differ diff --git a/dockerhubCrawler/views.py b/dockerhubCrawler/views.py index 058dafd..ed9ec7e 100644 --- a/dockerhubCrawler/views.py +++ b/dockerhubCrawler/views.py @@ -4,6 +4,9 @@ from .models import DockerhubCrawler # from django.db import models (Tried doesn't work:Sami) import datetime from datetime import timedelta +import os +from django.core.mail import send_mail +import pytz # Create your views here. @@ -54,6 +57,19 @@ def DockerPage(request): # end for here, string_obj should have latest pushed date by now splits = (string_obj.split(":")) + + newtime = datetimeobjtag_last_pushed.replace(tzinfo=pytz.UTC) + if dhubobj.last_pushed < newtime: + message = '"From DockerhubCrawler: '+ dhubobj.url + ' updated Image at: ' + str(datetimeobjtag_last_pushed)+ '"' + os.system('notipy send ' + str(message)) + send_mail( + dhubobj.url, + 'New Image @:'+ str (datetimeobjtag_last_pushed), + 'from@example.com', + ['to@example.com'], + fail_silently=False, + ) + dhubobj.last_pushed = datetimeobjtag_last_pushed # here we will notify and send the email dhubobj.save() diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..3099040 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +python manage.py migrate --no-input +python manage.py collectstatic --no-input +gunicorn crawlerApp.wsgi:application --bind 0.0.0.0:8000 diff --git a/githubCrawler/__pycache__/views.cpython-39.pyc b/githubCrawler/__pycache__/views.cpython-39.pyc index fe4eb03..6c7181b 100644 Binary files a/githubCrawler/__pycache__/views.cpython-39.pyc and b/githubCrawler/__pycache__/views.cpython-39.pyc differ diff --git a/githubCrawler/views.py b/githubCrawler/views.py index bf4e02a..7628100 100644 --- a/githubCrawler/views.py +++ b/githubCrawler/views.py @@ -5,6 +5,12 @@ from .models import GithubCrawler import requests from bs4 import BeautifulSoup import datetime +from django.core.mail import send_mail +import os + + + + def month_converter(month): months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] @@ -44,7 +50,15 @@ def GithubPage(request): if githubobj.last_pushed_number: if githubobj.last_pushed_number < last_pushed_number_thing: - print("send email and notify here") + message = '"From GitCrawler: '+ githubobj.url + ' updated Commit: ' + str(last_pushed_number_thing)+ '"' + os.system('notipy send ' + str(message)) + send_mail( + githubobj.url, + 'Commit '+ str (last_pushed_number_thing), + 'from@example.com', + ['to@example.com'], + fail_silently=False, + ) githubobj.last_pushed_number = last_pushed_number_thing githubobj.last_pushed = last_pushed_thing diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 0000000..ec37035 --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,3 @@ +FROM nginx:1.19.0-alpine + +COPY ./default.conf /etc/nginx/conf.d/default.conf \ No newline at end of file diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 0000000..962190f --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,15 @@ +upstream web { + server crawlerApp:8000; +} + +server { + listen 3000; + + location / { + proxy_pass http://web/; + } + location /static/ { + alias /app/static/; + } + +} \ No newline at end of file diff --git a/notipy b/notipy new file mode 100755 index 0000000..887e5b1 --- /dev/null +++ b/notipy @@ -0,0 +1,8 @@ +#!/usr/local/opt/python@3.9/bin/python3.9 +# -*- coding: utf-8 -*- +import re +import sys +from noti_py.noti import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/requirements.txt b/requirements.txt index 0c9d44a..67e08d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,32 +1,67 @@ -backoff==1.11.1 +wheel +aiofiles==0.6.0 +aiohttp==3.8.1 +aiohttp-socks==0.7.1 +aiosignal==1.2.0 +appdirs==1.4.4 +asgiref==3.5.0 +async-timeout==4.0.2 +asyncio==3.4.3 +atomicwrites==1.4.0 +attrs==21.4.0 beautifulsoup4==4.10.0 +bs4==0.0.1 +cachetools==5.0.0 certifi==2021.10.8 -charset-normalizer==2.0.10 +cffi==1.15.0 +charset-normalizer==2.0.11 click==8.0.3 coverage==5.5 -gpg==1.16.0 +cssselect==1.1.0 +DateTime==4.4 +Django==3.0 +fake-useragent==0.1.11 +frozenlist==1.3.0 +future==0.18.2 +h11==0.12.0 +h2==4.1.0 +hpack==4.0.0 +hyperframe==6.0.1 idna==3.3 +importlib-metadata==4.10.1 +jsonschema==3.2.0 +Logbook==1.5.3 lxml==4.7.1 +Markdown==3.3.6 +matrix-nio==0.19.0 +multidict==6.0.2 noti.py==0.4.0 -numpy==1.22.0 -pandas==1.3.5 -protobuf==3.19.2 -pyonfleet==1.2.1 -PyQt3D==5.15.5 -PyQt5==5.15.6 -PyQt5-sip==12.9.0 -PyQtChart==5.15.5 -PyQtDataVisualization==5.15.5 -PyQtNetworkAuth==5.15.5 -PyQtPurchasing==5.15.5 -PyQtWebEngine==5.15.5 -python-dateutil==2.8.2 +parse==1.19.0 +peewee==3.14.9 +Pillow==9.0.1 +pycparser==2.21 +pycryptodome==3.14.1 +pyee==8.2.2 +pyppeteer==1.0.2 +pyquery==1.4.3 +pyrsistent==0.18.1 +python-magic==0.4.25 +python-olm==3.1.3 +python-socks==2.0.3 pytz==2021.3 PyYAML==5.4.1 -ratelimit==2.2.1 requests==2.27.1 +requests-html==0.10.0 six==1.16.0 soupsieve==2.3.1 -TBB==0.2 +sqlparse==0.4.2 tqdm==4.62.3 +unpaddedbase64==2.1.0 urllib3==1.26.8 +w3lib==1.22.0 +websockets==10.1 +yarl==1.7.2 +zipp==3.7.0 +zope.interface==5.4.0 +gunicorn==20.1.0 +psycopg2-binary==2.8.6