From 73fc171761ab1f7317f2481638d9c976baec0980 Mon Sep 17 00:00:00 2001 From: Sami Date: Sun, 20 Feb 2022 23:11:15 +0500 Subject: [PATCH] docker and docker compose --- Dockerfile | 35 +++++++++ Makefile | 18 +++++ config.yaml | 8 ++ .../__pycache__/settings.cpython-39.pyc | Bin 2731 -> 2969 bytes crawlerApp/settings.py | 41 ++++++++-- db.sqlite3 | Bin 151552 -> 151552 bytes docker-compose.yml | 60 +++++++++++++++ .../__pycache__/views.cpython-39.pyc | Bin 1662 -> 2069 bytes .../migrations/0013_auto_20220220_0658.py | 25 ++++++ .../0013_auto_20220220_0658.cpython-39.pyc | Bin 0 -> 882 bytes dockerhubCrawler/views.py | 16 ++++ entrypoint.sh | 5 ++ .../__pycache__/views.cpython-39.pyc | Bin 1528 -> 1800 bytes githubCrawler/views.py | 16 +++- nginx/Dockerfile | 3 + nginx/default.conf | 15 ++++ notipy | 8 ++ requirements.txt | 71 +++++++++++++----- 18 files changed, 296 insertions(+), 25 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 config.yaml create mode 100644 docker-compose.yml create mode 100644 dockerhubCrawler/migrations/0013_auto_20220220_0658.py create mode 100644 dockerhubCrawler/migrations/__pycache__/0013_auto_20220220_0658.cpython-39.pyc create mode 100644 entrypoint.sh create mode 100644 nginx/Dockerfile create mode 100644 nginx/default.conf create mode 100755 notipy diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e2b5744 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.9-alpine + +ADD ./requirements.txt /app/requirements.txt + + +RUN set -ex \ + && apk add --no-cache --virtual .build-deps postgresql-dev build-base linux-headers jpeg-dev zlib-dev gcc musl-dev python3-dev libffi-dev openssl-dev olm-dev \ + && python -m venv /env \ + && /env/bin/pip install --upgrade pip \ + && /env/bin/pip install -U setuptools \ + && /env/bin/pip install wheel \ + && /env/bin/pip install --no-cache-dir -r /app/requirements.txt \ + && runDeps="$(scanelf --needed --nobanner --recursive /env \ + | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ + | sort -u \ + | xargs -r apk info --installed \ + | sort -u)" \ + && apk add --virtual rundeps $runDeps \ + && apk del .build-deps + +ADD ./ /app +WORKDIR /app + +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH +RUN pip install noti_py + +COPY ./entrypoint.sh / +ENTRYPOINT [ "sh", "entrypoint.sh" ] + + + + + + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bf0a7c3 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +tag=latest +image=crawlerapp +build: + docker build --force-rm $(options) -t crawlerapp:latest . + +compose-start: + docker-compose up --remove-orphans $(options) + +compose-stop: + docker-compose down --remove-orphans $(options) + +compose-manage-py: + docker-compose run --rm $(options) website /env/bin/python /app/manage.py createsuperuser + +migrate: + /env/bin/python /app/manage.py migrate +start-server: + gunicorn --bind :8000 --error-logfile /var/log/gunicorn-error.log --workers 3 crawlerApp.wsgi:application \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..4afe282 --- /dev/null +++ b/config.yaml @@ -0,0 +1,8 @@ +homeserver: + base: https://ungleich.matrix.ungleich.cloud + api_base: /_matrix/client/r0 +user: + token: syt_YXNhbWloYXNzYW4_KVTnvQNvKmDKmYvYQwMF_2vLBFx +room: + id: + - '!FCoFjcLuSBdTOQzVaG:ungleich.ch' diff --git a/crawlerApp/__pycache__/settings.cpython-39.pyc b/crawlerApp/__pycache__/settings.cpython-39.pyc index 8624a2bc86bcce86e1a99d481f4cc133f1174c5e..db7763b4bc38f929c2cbc7230ed0833aa1d38d3c 100644 GIT binary patch delta 827 zcmYjO%TC)s6!qAH<2ZI6gv2}&LU^YsVTUsRm7qob=4)yenS7i+phWxEK0|bNtvtrIOkrUJJ$TjebuTRRh2n*j$5PMUm1@3 z9fF5PVqk;0`|(S)dQM{$^`u@L1i&!ziT{9zOcN0&LM-AGkW55WlY{s*B1x8ImQ#=* zX;9cS|1CpBP_K9sK@w7|pT0&Gg3Kjm-;Kg3$wC&!hLgD~9`cvGb&g0#h8NIaJX}U{ z5uS)}KEjg`)*?I=;c==!?2!MA9BHsZgq+3=#F@68XIz2iXr5|Nq~kDsDO!w&63npc zmAX(46U+|coP|tGCSaaS!UCCsBq=a2GM|P;QeuC&sBvV5Ds0&SERnKZfMrsK3YmpR zWDcri9v+hgHopifWC?hktJH4R&>C6^WWB!KtQ+m;^_N6C?VmX9LwZ_4K@8|Y=j6x> zMt163t=&I}JiUaDbLXrGy^d}6YtXHAJNt*!2Df&vb$n#4=Cat<&$FufV{qM00wCrz4>(hekIYErtZ!D*82ZBjGhY$4x z%k~`XUr4<%M`eVQ3_~gmX%>7Of2AaJW7BxH-EKA-hVR5(bvEBx_o&^;aTNG+0js4o+gpmN(rj&;@swVmatR zuO6B`*)+!X>XF31!oPs47mfdc2X)Y-`d+^G=F5EFyq9?w`xwb1P{dmr?8 zzD&sfAuw{E!4{A6`B%Ubfr=Eh{4EJEc)|CUdyR+;f{sW)_kehgh{C#`^&m)O3S_3{ zK7^v?cgUBgoATk3{h%sdp=2&e;Vg3(U zFp!SPEW}9!5+n)=i80JE%tMkaTs*Wm8A;+)w!T7&B+LY)Ndht?2}@)VvLpqUNSe7a zaG5LtSM}tUN1xCp+8(YQtqu2_MpobJ!unu6m%DK*d$hT_veCa;XxY1&!p4ik!RqsS zo$K_#%=MNFbbsvKDnF<{`i9td<JvA;2w)7DC=JJ-%d-_a%rbrYiyUFj(Nj$#sW1D4XbO8d7U~I=f3f`Tuyox z4pnQ#TBY=`@}yjCY*luuZdH>!u4$|N-k@u> y8$GMrqD{wEo0c}{P^;*EmsSG1%!>mepAfLuP03lLxQ3jJ;J`18G4k;k{mb9Eh?YM9 diff --git a/crawlerApp/settings.py b/crawlerApp/settings.py index 556b99c..718ed5c 100644 --- a/crawlerApp/settings.py +++ b/crawlerApp/settings.py @@ -20,8 +20,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'y2zbt_#gbod3w3))59#y8*(6u7-js^!-6xvw*q=l4emn)p&-ef' -#SECRET_KEY = str(os.environ.get('SECRET_KEY',default="your_key")) +# SECRET_KEY = 'y2zbt_#gbod3w3))59#y8*(6u7-js^!-6xvw*q=l4emn)p&-ef' +SECRET_KEY = str(os.environ.get('SECRET_KEY',default="your_key")) # SECURITY WARNING: don't run with debug turned on in production! #DEBUG = True @@ -85,6 +85,35 @@ DATABASES = { } +# DBNam = os.getenv('DBNam') +POSTGRES_HOST = os.environ.get('POSTGRES_HOST',default='postgres') +POSTGRES_DB = os.environ.get('POSTGRES_DB',default='postgres') +POSTGRES_USER = os.environ.get('POSTGRES_USER',default='postgres') +POSTGRES_PASSWORD = os.environ.get('POSTGRES_PASSWORD',default='postgres') + +if DEBUG == 1: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR + 'db.sqlite3', + } + } +else: + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'HOST': POSTGRES_HOST, # + 'NAME': POSTGRES_DB, # btredb local + 'USER': POSTGRES_USER, + 'PORT': 5432, + 'PASSWORD': POSTGRES_PASSWORD, + }} + + + + + # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -122,11 +151,11 @@ USE_TZ = True # https://docs.djangoproject.com/en/3.0/howto/static-files/ STATIC_URL = '/static/' -# STATIC_ROOT = BASE_DIR + '/static/' +STATIC_ROOT = BASE_DIR + '/static/' -STATICFILES_DIRS = [ - os.path.join(BASE_DIR, "static"), -] +# STATICFILES_DIRS = [ +# os.path.join(BASE_DIR, "static"), +# ] if DEBUG: EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' diff --git a/db.sqlite3 b/db.sqlite3 index 63c6c1d8a5c7c52cb6dcee73f23bedc520558e6c..cacc16d27ad2c06c8b85ada3df6d7f519dd51412 100644 GIT binary patch delta 293 zcmZozz}c{XbAmME&WSS4j5{|bWaTL&=a!Tkm022>8kiVZmRJ}hnpBvW=Vjy=nHW@= zmX;cCewD|>C}ON@V4`4XZe?s@Wn`phVq|P-WU#5=0Y5?Qn;K5Z3mIA<>oqboHgCQ& zfBT*Ji~$I{BkK+nJe2+XJN<7HZdML5xqNqBl6AJbG^a_;;PIa4bG o3o8>dJ!1<~GYg~bZ~2+Z__=e942+C)4UBXREw<;$Gfgr808dU(VE_OC delta 254 zcmZozz}c{XbAmME!ih4@j0-m=WaTMTmSiWDC}N~*WT0SZWMyn*WoW5qVrpSzX|}200Y5?Qn;K5Z3mKRp>oqg7G;F>z zfBT*Ji~hZY3iFBO_e{BV9w|?RoM{lMDc<3rr0F diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1d0190e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,60 @@ +version: '3.7' + +x-service-volumes: &service-volumes + - ./:/app/:rw,cached + - static:/app/static +x-database-variables: &database-variables + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + + +x-app-variables: &app-variables + <<: *database-variables + POSTGRES_HOST: postgres + DEBUG : 0 + + +services: + crawlerApp: + image: crawlerapp:latest + # command: ["gunicorn", "--bind", ":8000","--error-logfile" ,"/var/log/gunicorn-error.log","--workers", "1", "crawlerApp.wsgi:application"] + # expose: + # - 8000 + # #command: /env/bin/python /app/manage.py runserver 0.0.0.0:3000 + volumes: *service-volumes + environment: *app-variables + depends_on: + - db_migrate + ports: + - "8000:8000" + + db_migrate: + image: crawlerapp:latest + command: /env/bin/python /app/manage.py migrate + volumes: *service-volumes + environment: *app-variables + depends_on: + - postgres + + nginx: + build : ./nginx + ports: + - '3000:3000' + volumes: *service-volumes + depends_on: + - crawlerApp + + + + postgres: + image: postgres + ports: + - "5432:5432" + environment: *app-variables + volumes: + - db-data:/var/lib/postgresql/data + +volumes: + db-data: + static: \ No newline at end of file diff --git a/dockerhubCrawler/__pycache__/views.cpython-39.pyc b/dockerhubCrawler/__pycache__/views.cpython-39.pyc index 513bcfd21d1ef0b195f530325cd8acfe84e2d00e..d380ec26b886a15def5bf0d2323db1cef3a41513 100644 GIT binary patch delta 772 zcmYjPTWb?R6rMA?*=+WfOE0Ol$`T4%m4Xiz2_l6`K_80O2SXujGSfEcZZ^(LY_n|X zgT7V_i|_3p@WlroeCe<7VgG}FKtY^Ii*(?8bIzG>&W-ce`BL!$*R_y5zw&bDMS0>a z0h!!=cpniQ7F$Jvh!L7msZ|0bbsSnPixi3Z5w&bmB9^=zf?uH(dvdtuu0USmbU@ls z&MAPeN$ zNdZubQ@veI_0zdC1n6Bu;J$XMpP?g1%%@dE7RmM0=>b_f(PWP|vIqLxW7(u3lHOn^ zP8J%QEROsqad)4xy-{a_1qUHzO+T6OM?(?_O8lo$uuJ_wH2vSuNSr||dc(1=rsOBi z7CpEmtT&U&jtsU=4})kJ($#JpC7y_{+M6=9>+ZX_dB(zVL&Y$nn9z{3f;8Ao1%`rE^;RMMV6Yp@fiSwN7He8EyQ2td kd8Wax6g4exTABGsh8>0UvD!CYli+{}4wj;BFm+S^2b5mEkN^Mx delta 361 zcmbO#@Q;Ttk(ZZ?0SMZ5|4%%}Gm%e*(PN^vg(zDJdkaGpdkRMiCy?eyVG3r@!R%}*Hpm~?7b z;(2OV7Vs`)U}Pv%Dq+fEUcj=Dp_Z$JuZAH-AVsi-E0ZxrsFxXNq44BGERmC!vuZH% zPrkq!B+Z?ZSX>fcP+FXknsSS!II%3Xh#6>sCU+6%OS#mvQAqzy8Gb8;QKM1&kjHCt|eN@`AV5i5{W tBn2YaKm^E(V6{l5gY5xZ!C?c`RhpA(2MUB@kXjx_4n`g(AZFoU0RUP`N@@TA diff --git a/dockerhubCrawler/migrations/0013_auto_20220220_0658.py b/dockerhubCrawler/migrations/0013_auto_20220220_0658.py new file mode 100644 index 0000000..8bfb788 --- /dev/null +++ b/dockerhubCrawler/migrations/0013_auto_20220220_0658.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0 on 2022-02-20 06:58 + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('dockerhubCrawler', '0012_auto_20220213_1751'), + ] + + operations = [ + migrations.AlterField( + model_name='dockerhubcrawler', + name='last_pushed', + field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 27, 6, 58, 29, 116932, tzinfo=utc), null=True), + ), + migrations.AlterField( + model_name='dockerhubcrawler', + name='last_updated', + field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 27, 6, 58, 29, 116976, tzinfo=utc), null=True), + ), + ] diff --git a/dockerhubCrawler/migrations/__pycache__/0013_auto_20220220_0658.cpython-39.pyc b/dockerhubCrawler/migrations/__pycache__/0013_auto_20220220_0658.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0bf3eedf3f8aa832be9cd65dceba1b4d5e1a2968 GIT binary patch literal 882 zcmZ`%&2G~`5Z?7)94DbE5<&tB7cRNfNvZ-u2(<#iiC!odUsjH1o7!FfuxnG%D_ncz zA$SiCJi}fqA?3!o5`tMLRijGmddD-PZ)Rt{op>+^2(E9xHfLKlAs5D2Uk)03c+_(g zK!6GonZG4yZbepZM>aLKEu7qqTuR;&aKQaaz~xrveX%1C=zB8q78p+sMwSlpbS~pM zEeoZ+yaX=Ph-$m3lf*#Q*AR_8Jn9n)P9h2ibe?Z2Q7!Jh;xoUBi*Bv3()=cwx)u(=8*oz zd=ud}9(zoD-r4uQBam zn)BADeRE5VrZf|ygR)=FJPr4&N~Fnh6vC!DE#(QTWSQ}# zR^f^>#Dq5~53eUEMDlmm$)Aqd_{r|G?dr2M)cSbTJ)_vxU&nQ*h5PZJHSMy+jXLrH zqI4IsxR{sQFl*axTaE6loK2kywQZU_D~n~LZSZw}OYM=#M%`ndw^T6)pN#QK0zUR z1W(s;RF)^=;-G?GBD)Q-_wU4Mm840+-&01gJ{(IEcAtHKCm8*NUx%9^^yfo;mm>Zq z&Xc!LWJn?wL;VfVeX+<63{f~PVu4~t9Gd_zQHrrxJj0c++y$&)^(YA})I->s4^`6T z5RNMo2JE@^|Nn!msD=z{cu^0tJ-|yxGIhEXU6*$cuMpW2SPk;aE#Gs^2R+gBt$oM# z8)k699PXhdY&4sm>-I#$Jb~pvA9$j-H!|Y^nL+9_Wd?IQ^nTrbVYz#by|V4OK~{L@ z{T5Bt*r9=L@t$K3gfj{<-RP^l47KRH+=Wu~Fmbyg4}E7UbKy@Vk53go;=*=)g;+)} z6J^Ln9}^p}6#Yotxar!QTU~pavK+_j^toRk)Tt`cZK_c-&xW1Hgjpb;x1xK=)yoyg zg9HiCW?K5As0t{OCTX(YB2?XoUMEX%E&80SZr_T_%wgXebUkXxw^!nYn6dF2l$-6q j?Af`$XDns})Vr>SL&si=w}j_O>9jbyL=sz26*c)6nyaR@ delta 435 zcmYk2%}T>S5XX16P28kuP};OwMXDlLdZ-}iMT(#xf(M~DL2By8eno4usiMS_9z+mY zR`4X~oA?-_52AQ-wkkLFc zH!vHVR!G4d+^-B?AHb0(uwOQ?NyJqvFbJ}k4?vxM&Bn~o*U`l`Dj{jX6MZHdqz`+u z(xx9XR$ZF$FkPAn7>I#5FRs_{l#4x5KcygWzkgoaHeVGh15#ed>M9C)b%v}s0 zmX-Zjdu|pdeL?9WC%j{x_m14ApLel3?u5<5^L)b#Jb4zMhpc>z*D-3^(nP{{MOZ!Q ziH67D;=(=k0*aV|Eq$#5)AB`ITyTBAQ)`L9LE$H4uzE8$n_;6@+i~lj`m0!$GkR%^ zQ5|$@QZp8m708rbeH;q%R-ariDS%qkQ2NIkl7~>Xdeq@P-z%$C3dOZ>yB0+UMw#{v D4>4O$ diff --git a/githubCrawler/views.py b/githubCrawler/views.py index bf4e02a..7628100 100644 --- a/githubCrawler/views.py +++ b/githubCrawler/views.py @@ -5,6 +5,12 @@ from .models import GithubCrawler import requests from bs4 import BeautifulSoup import datetime +from django.core.mail import send_mail +import os + + + + def month_converter(month): months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] @@ -44,7 +50,15 @@ def GithubPage(request): if githubobj.last_pushed_number: if githubobj.last_pushed_number < last_pushed_number_thing: - print("send email and notify here") + message = '"From GitCrawler: '+ githubobj.url + ' updated Commit: ' + str(last_pushed_number_thing)+ '"' + os.system('notipy send ' + str(message)) + send_mail( + githubobj.url, + 'Commit '+ str (last_pushed_number_thing), + 'from@example.com', + ['to@example.com'], + fail_silently=False, + ) githubobj.last_pushed_number = last_pushed_number_thing githubobj.last_pushed = last_pushed_thing diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 0000000..ec37035 --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,3 @@ +FROM nginx:1.19.0-alpine + +COPY ./default.conf /etc/nginx/conf.d/default.conf \ No newline at end of file diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 0000000..962190f --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,15 @@ +upstream web { + server crawlerApp:8000; +} + +server { + listen 3000; + + location / { + proxy_pass http://web/; + } + location /static/ { + alias /app/static/; + } + +} \ No newline at end of file diff --git a/notipy b/notipy new file mode 100755 index 0000000..887e5b1 --- /dev/null +++ b/notipy @@ -0,0 +1,8 @@ +#!/usr/local/opt/python@3.9/bin/python3.9 +# -*- coding: utf-8 -*- +import re +import sys +from noti_py.noti import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/requirements.txt b/requirements.txt index 0c9d44a..67e08d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,32 +1,67 @@ -backoff==1.11.1 +wheel +aiofiles==0.6.0 +aiohttp==3.8.1 +aiohttp-socks==0.7.1 +aiosignal==1.2.0 +appdirs==1.4.4 +asgiref==3.5.0 +async-timeout==4.0.2 +asyncio==3.4.3 +atomicwrites==1.4.0 +attrs==21.4.0 beautifulsoup4==4.10.0 +bs4==0.0.1 +cachetools==5.0.0 certifi==2021.10.8 -charset-normalizer==2.0.10 +cffi==1.15.0 +charset-normalizer==2.0.11 click==8.0.3 coverage==5.5 -gpg==1.16.0 +cssselect==1.1.0 +DateTime==4.4 +Django==3.0 +fake-useragent==0.1.11 +frozenlist==1.3.0 +future==0.18.2 +h11==0.12.0 +h2==4.1.0 +hpack==4.0.0 +hyperframe==6.0.1 idna==3.3 +importlib-metadata==4.10.1 +jsonschema==3.2.0 +Logbook==1.5.3 lxml==4.7.1 +Markdown==3.3.6 +matrix-nio==0.19.0 +multidict==6.0.2 noti.py==0.4.0 -numpy==1.22.0 -pandas==1.3.5 -protobuf==3.19.2 -pyonfleet==1.2.1 -PyQt3D==5.15.5 -PyQt5==5.15.6 -PyQt5-sip==12.9.0 -PyQtChart==5.15.5 -PyQtDataVisualization==5.15.5 -PyQtNetworkAuth==5.15.5 -PyQtPurchasing==5.15.5 -PyQtWebEngine==5.15.5 -python-dateutil==2.8.2 +parse==1.19.0 +peewee==3.14.9 +Pillow==9.0.1 +pycparser==2.21 +pycryptodome==3.14.1 +pyee==8.2.2 +pyppeteer==1.0.2 +pyquery==1.4.3 +pyrsistent==0.18.1 +python-magic==0.4.25 +python-olm==3.1.3 +python-socks==2.0.3 pytz==2021.3 PyYAML==5.4.1 -ratelimit==2.2.1 requests==2.27.1 +requests-html==0.10.0 six==1.16.0 soupsieve==2.3.1 -TBB==0.2 +sqlparse==0.4.2 tqdm==4.62.3 +unpaddedbase64==2.1.0 urllib3==1.26.8 +w3lib==1.22.0 +websockets==10.1 +yarl==1.7.2 +zipp==3.7.0 +zope.interface==5.4.0 +gunicorn==20.1.0 +psycopg2-binary==2.8.6