docker and docker compose

Dockerfile

@@ -0,0 +1,35 @@
+FROM python:3.9-alpine
+
+ADD ./requirements.txt /app/requirements.txt
+
+
+RUN set -ex \
+    && apk add --no-cache --virtual .build-deps postgresql-dev build-base linux-headers jpeg-dev zlib-dev gcc musl-dev python3-dev libffi-dev openssl-dev olm-dev \
+    && python -m venv /env \
+    && /env/bin/pip install --upgrade pip \
+    && /env/bin/pip install -U setuptools \
+    && /env/bin/pip install wheel \
+    && /env/bin/pip install --no-cache-dir -r /app/requirements.txt \
+    && runDeps="$(scanelf --needed --nobanner --recursive /env \
+        | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \
+        | sort -u \
+        | xargs -r apk info --installed \
+        | sort -u)" \
+    && apk add --virtual rundeps $runDeps \
+    && apk del .build-deps
+
+ADD ./ /app
+WORKDIR /app
+
+ENV VIRTUAL_ENV /env
+ENV PATH /env/bin:$PATH
+RUN pip install noti_py
+
+COPY ./entrypoint.sh /
+ENTRYPOINT [ "sh", "entrypoint.sh" ]
+
+
+
+
+
+

Makefile

@@ -0,0 +1,18 @@
+tag=latest
+image=crawlerapp
+build:
+	docker build --force-rm $(options) -t crawlerapp:latest .
+
+compose-start:
+	docker-compose up --remove-orphans $(options)
+
+compose-stop:
+	docker-compose down --remove-orphans $(options)
+
+compose-manage-py:
+	docker-compose run --rm $(options) website /env/bin/python /app/manage.py createsuperuser
+
+migrate:
+	/env/bin/python /app/manage.py migrate
+start-server:
+	gunicorn --bind :8000 --error-logfile /var/log/gunicorn-error.log --workers 3 crawlerApp.wsgi:application

config.yaml

@@ -0,0 +1,8 @@
+homeserver:
+    base: https://ungleich.matrix.ungleich.cloud
+    api_base: /_matrix/client/r0
+user:
+    token: syt_YXNhbWloYXNzYW4_KVTnvQNvKmDKmYvYQwMF_2vLBFx
+room:
+    id:
+    - '!FCoFjcLuSBdTOQzVaG:ungleich.ch'

crawlerApp/settings.py

@@ -20,8 +20,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'y2zbt_#gbod3w3))59#y8*(6u7-js^!-6xvw*q=l4emn)p&-ef'
-#SECRET_KEY = str(os.environ.get('SECRET_KEY',default="your_key"))
+# SECRET_KEY = 'y2zbt_#gbod3w3))59#y8*(6u7-js^!-6xvw*q=l4emn)p&-ef'
+SECRET_KEY = str(os.environ.get('SECRET_KEY',default="your_key"))
 
 # SECURITY WARNING: don't run with debug turned on in production!
 #DEBUG = True
@@ -85,6 +85,35 @@ DATABASES = {
 }
 
 
+# DBNam = os.getenv('DBNam')
+POSTGRES_HOST = os.environ.get('POSTGRES_HOST',default='postgres')
+POSTGRES_DB = os.environ.get('POSTGRES_DB',default='postgres')
+POSTGRES_USER = os.environ.get('POSTGRES_USER',default='postgres')
+POSTGRES_PASSWORD = os.environ.get('POSTGRES_PASSWORD',default='postgres')
+
+if DEBUG == 1:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': BASE_DIR + 'db.sqlite3',
+        }
+    }
+else:
+
+    DATABASES = {
+            'default': {
+                'ENGINE': 'django.db.backends.postgresql',
+                'HOST': POSTGRES_HOST, # 
+                'NAME': POSTGRES_DB, # btredb local
+                'USER': POSTGRES_USER,
+                'PORT': 5432,
+                'PASSWORD': POSTGRES_PASSWORD,
+            }}
+
+
+
+
+
 # Password validation
 # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
 
@@ -122,11 +151,11 @@ USE_TZ = True
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 
 STATIC_URL = '/static/'
-# STATIC_ROOT = BASE_DIR + '/static/'
+STATIC_ROOT = BASE_DIR + '/static/'
 
-STATICFILES_DIRS = [
-    os.path.join(BASE_DIR, "static"),
-]
+# STATICFILES_DIRS = [
+#     os.path.join(BASE_DIR, "static"),
+# ]
 
 if DEBUG:
     EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

docker-compose.yml

@@ -0,0 +1,60 @@
+version: '3.7'
+
+x-service-volumes: &service-volumes
+  - ./:/app/:rw,cached
+  - static:/app/static
+x-database-variables: &database-variables
+  POSTGRES_DB: postgres
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+
+
+x-app-variables: &app-variables
+  <<: *database-variables
+  POSTGRES_HOST: postgres
+  DEBUG : 0
+
+
+services:
+  crawlerApp: 
+    image: crawlerapp:latest
+    # command: ["gunicorn", "--bind", ":8000","--error-logfile" ,"/var/log/gunicorn-error.log","--workers", "1", "crawlerApp.wsgi:application"]
+    # expose:
+    #   - 8000
+    # #command: /env/bin/python /app/manage.py runserver 0.0.0.0:3000
+    volumes: *service-volumes
+    environment: *app-variables
+    depends_on:
+      - db_migrate
+    ports:
+      - "8000:8000"
+
+  db_migrate:
+    image:  crawlerapp:latest
+    command: /env/bin/python /app/manage.py migrate
+    volumes: *service-volumes
+    environment: *app-variables
+    depends_on:
+      - postgres
+
+  nginx:
+      build : ./nginx
+      ports:
+        - '3000:3000'
+      volumes: *service-volumes
+      depends_on:
+        - crawlerApp
+
+
+
+  postgres:
+    image: postgres
+    ports:
+      - "5432:5432"
+    environment: *app-variables
+    volumes:
+      - db-data:/var/lib/postgresql/data
+
+volumes:
+  db-data:
+  static: 

dockerhubCrawler/migrations/0013_auto_20220220_0658.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0 on 2022-02-20 06:58
+
+import datetime
+from django.db import migrations, models
+from django.utils.timezone import utc
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dockerhubCrawler', '0012_auto_20220213_1751'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_pushed',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 27, 6, 58, 29, 116932, tzinfo=utc), null=True),
+        ),
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_updated',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 27, 6, 58, 29, 116976, tzinfo=utc), null=True),
+        ),
+    ]

dockerhubCrawler/views.py

@@ -4,6 +4,9 @@ from .models import DockerhubCrawler
 # from django.db import models (Tried doesn't work:Sami)
 import datetime
 from datetime import timedelta
+import os
+from django.core.mail import send_mail
+import pytz
 
 # Create your views here.
 
@@ -54,6 +57,19 @@ def DockerPage(request):
             # end for here, string_obj should have latest pushed date by now      
         
         splits = (string_obj.split(":"))
+
+        newtime = datetimeobjtag_last_pushed.replace(tzinfo=pytz.UTC)
+        if dhubobj.last_pushed < newtime:
+            message = '"From DockerhubCrawler: '+ dhubobj.url + ' updated Image at: ' + str(datetimeobjtag_last_pushed)+ '"' 
+            os.system('notipy send ' + str(message))
+            send_mail(
+                        dhubobj.url,
+                        'New Image @:'+ str (datetimeobjtag_last_pushed),
+                        'from@example.com',
+                        ['to@example.com'],
+                        fail_silently=False,
+                    )
+
         dhubobj.last_pushed = datetimeobjtag_last_pushed
         # here we will notify and send the email
         dhubobj.save()

entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+python manage.py migrate --no-input
+python manage.py collectstatic --no-input
+gunicorn crawlerApp.wsgi:application --bind 0.0.0.0:8000

githubCrawler/views.py

@@ -5,6 +5,12 @@ from .models import GithubCrawler
 import requests
 from bs4 import BeautifulSoup
 import datetime
+from django.core.mail import send_mail
+import os
+
+ 
+
+
 
 def month_converter(month):
     months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
@@ -44,7 +50,15 @@ def GithubPage(request):
 
         if githubobj.last_pushed_number:
             if githubobj.last_pushed_number < last_pushed_number_thing:
-                print("send email and notify here")
+                message = '"From GitCrawler: '+ githubobj.url + ' updated Commit: ' + str(last_pushed_number_thing)+ '"' 
+                os.system('notipy send ' + str(message))
+                send_mail(
+                            githubobj.url,
+                            'Commit '+ str (last_pushed_number_thing),
+                            'from@example.com',
+                            ['to@example.com'],
+                            fail_silently=False,
+                        )
         
         githubobj.last_pushed_number = last_pushed_number_thing
         githubobj.last_pushed = last_pushed_thing

nginx/Dockerfile

@@ -0,0 +1,3 @@
+FROM nginx:1.19.0-alpine
+
+COPY ./default.conf /etc/nginx/conf.d/default.conf

nginx/default.conf

@@ -0,0 +1,15 @@
+upstream web {  
+  server crawlerApp:8000;
+}
+
+server {
+    listen 3000;
+    
+    location / {
+        proxy_pass http://web/;
+    }
+    location /static/ {    
+        alias /app/static/; 
+    }
+
+}

notipy

@@ -0,0 +1,8 @@
+#!/usr/local/opt/python@3.9/bin/python3.9
+# -*- coding: utf-8 -*-
+import re
+import sys
+from noti_py.noti import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

requirements.txt

@@ -1,32 +1,67 @@
-backoff==1.11.1
+wheel
+aiofiles==0.6.0
+aiohttp==3.8.1
+aiohttp-socks==0.7.1
+aiosignal==1.2.0
+appdirs==1.4.4
+asgiref==3.5.0
+async-timeout==4.0.2
+asyncio==3.4.3
+atomicwrites==1.4.0
+attrs==21.4.0
 beautifulsoup4==4.10.0
+bs4==0.0.1
+cachetools==5.0.0
 certifi==2021.10.8
-charset-normalizer==2.0.10
+cffi==1.15.0
+charset-normalizer==2.0.11
 click==8.0.3
 coverage==5.5
-gpg==1.16.0
+cssselect==1.1.0
+DateTime==4.4
+Django==3.0
+fake-useragent==0.1.11
+frozenlist==1.3.0
+future==0.18.2
+h11==0.12.0
+h2==4.1.0
+hpack==4.0.0
+hyperframe==6.0.1
 idna==3.3
+importlib-metadata==4.10.1
+jsonschema==3.2.0
+Logbook==1.5.3
 lxml==4.7.1
+Markdown==3.3.6
+matrix-nio==0.19.0
+multidict==6.0.2
 noti.py==0.4.0
-numpy==1.22.0
-pandas==1.3.5
-protobuf==3.19.2
-pyonfleet==1.2.1
-PyQt3D==5.15.5
-PyQt5==5.15.6
-PyQt5-sip==12.9.0
-PyQtChart==5.15.5
-PyQtDataVisualization==5.15.5
-PyQtNetworkAuth==5.15.5
-PyQtPurchasing==5.15.5
-PyQtWebEngine==5.15.5
-python-dateutil==2.8.2
+parse==1.19.0
+peewee==3.14.9
+Pillow==9.0.1
+pycparser==2.21
+pycryptodome==3.14.1
+pyee==8.2.2
+pyppeteer==1.0.2
+pyquery==1.4.3
+pyrsistent==0.18.1
+python-magic==0.4.25
+python-olm==3.1.3
+python-socks==2.0.3
 pytz==2021.3
 PyYAML==5.4.1
-ratelimit==2.2.1
 requests==2.27.1
+requests-html==0.10.0
 six==1.16.0
 soupsieve==2.3.1
-TBB==0.2
+sqlparse==0.4.2
 tqdm==4.62.3
+unpaddedbase64==2.1.0
 urllib3==1.26.8
+w3lib==1.22.0
+websockets==10.1
+yarl==1.7.2
+zipp==3.7.0
+zope.interface==5.4.0
+gunicorn==20.1.0
+psycopg2-binary==2.8.6