Github section working too, notifications and dockerfile rem

githubCrawler/migrations/0003_auto_20220213_1722.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.0 on 2022-02-13 17:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('githubCrawler', '0002_auto_20220205_1704'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='githubcrawler',
+            name='last_updated',
+        ),
+        migrations.AddField(
+            model_name='githubcrawler',
+            name='last_pushed_number',
+            field=models.IntegerField(null=True),
+        ),
+    ]

githubCrawler/migrations/0004_auto_20220213_1724.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0 on 2022-02-13 17:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('githubCrawler', '0003_auto_20220213_1722'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='api_url',
+            field=models.CharField(blank=True, max_length=1000, null=True),
+        ),
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='last_pushed',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='last_pushed_number',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

githubCrawler/migrations/0005_auto_20220213_1751.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2022-02-13 17:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('githubCrawler', '0004_auto_20220213_1724'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='last_pushed',
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]

githubCrawler/models.py

@@ -1,3 +1,4 @@
+from re import T
 from django.db import models
 
 # Create your models here.
@@ -5,18 +6,18 @@ from django.db import models
 
 class GithubCrawler(models.Model):
     url = models.CharField(max_length=1000,blank = False)
-    api_url = models.CharField(max_length=1000, null= True)
-    last_pushed = models.DateTimeField(null= True)
-    last_updated = models.DateTimeField(null= True)
+    api_url = models.CharField(max_length=1000, null= True, blank=True)
+    last_pushed = models.DateField(null= True,blank=True)
+    last_pushed_number = models.IntegerField(null= True,blank=True)
     def __str__(self):
         return self.url
 
     def save(self, *args, **kwargs):
-        split_string =  self.url.split("/")
-        self.api_url =  'https://hub.docker.com/v2/repositories/' + split_string[5] +'/' \
-                        + split_string[6] + '/tags/?page=1&page_size=800'
+        check_len = len(self.url)
+        if self.url[check_len-1] == "/":
+            self.api_url =  self.url + 'commits/master'
+        else:
+            self.api_url =  self.url + '/commits/master'
         super(GithubCrawler, self).save(*args, **kwargs)
 
 # Research , via Chrome dev tools ;)
-#https://hub.docker.com/r/vectorim/element-web/tags
-#https://hub.docker.com/v2/repositories/vectorim/element-web/tags/?page=1&page_size=800

githubCrawler/urls.py

@@ -1,11 +1,10 @@
 from django.urls import path
+from . import views
 
 
 
 
 urlpatterns = [
-#path('create_routes/<str:string_passed>/<str:start_date>/<str:end_date>/',views.create_routes, name='create_routes'),
-#path('check_LoggedIn/',views.logged_in_user, name='logged_in_user'),
-#path('check_Socials/',views.social_media_details, name='social_media_details'),
+path('github',views.GithubPage, name='GithubPage'),
 
 ]

githubCrawler/views.py

@@ -1,3 +1,57 @@
 from django.shortcuts import render
-
+import requests
+from .models import GithubCrawler
 # Create your views here.
+import requests
+from bs4 import BeautifulSoup
+import datetime
+
+def month_converter(month):
+    months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+    return months.index(month) + 1
+
+
+def GithubPage(request):
+
+    for githubobj in GithubCrawler.objects.all():
+
+        URL = githubobj.api_url
+        page = requests.get(URL)
+        temp = ''
+        Soup = BeautifulSoup(page.text, 'lxml')
+        heading_tags = ["h2"]
+        for tags in Soup.find_all(heading_tags):
+            # print(tags)
+            temp =  tags.text.strip()
+            break # first one is latest no need to loop through all of them
+
+        temp = temp[10:]
+        temp = temp.replace(",", "")
+        temp = temp.split(' ')
+        #print(temp)
+        last_pushed_thing = datetime.datetime(int(temp[3]), int(month_converter(temp[1])), int(temp[2])) # y-m-d
+
+        for div in Soup.findAll('div', attrs={'class':'issue-link js-issue-link'}):
+            print(div.find('a')['href'])
+
+        # finding commit number here and saving in db
+
+        el = Soup.find(class_='issue-link js-issue-link', href=True)
+        temp_result = (el['href'])
+
+        temp_result = temp_result.split('/')
+        last_pushed_number_thing = int(temp_result[len(temp_result)-1])
+
+        if githubobj.last_pushed_number:
+            if githubobj.last_pushed_number < last_pushed_number_thing:
+                print("send email and notify here")
+        
+        githubobj.last_pushed_number = last_pushed_number_thing
+        githubobj.last_pushed = last_pushed_thing
+
+        githubobj.save()
+
+    allobjs = GithubCrawler.objects.all()
+
+    
+    return render(request, 'github.html', context={'githubobjs': allobjs})