Github section working too, notifications and dockerfile rem

dockerhubCrawler/migrations/0010_auto_20220213_1722.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0 on 2022-02-13 17:22
+
+import datetime
+from django.db import migrations, models
+from django.utils.timezone import utc
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dockerhubCrawler', '0009_auto_20220206_0905'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_pushed',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 20, 17, 22, 49, 233471, tzinfo=utc), null=True),
+        ),
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_updated',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 20, 17, 22, 49, 233546, tzinfo=utc), null=True),
+        ),
+    ]

dockerhubCrawler/migrations/0011_auto_20220213_1724.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0 on 2022-02-13 17:24
+
+import datetime
+from django.db import migrations, models
+from django.utils.timezone import utc
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dockerhubCrawler', '0010_auto_20220213_1722'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_pushed',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 20, 17, 24, 29, 7546, tzinfo=utc), null=True),
+        ),
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_updated',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 20, 17, 24, 29, 7578, tzinfo=utc), null=True),
+        ),
+    ]

dockerhubCrawler/migrations/0012_auto_20220213_1751.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0 on 2022-02-13 17:51
+
+import datetime
+from django.db import migrations, models
+from django.utils.timezone import utc
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dockerhubCrawler', '0011_auto_20220213_1724'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_pushed',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 20, 17, 51, 59, 100227, tzinfo=utc), null=True),
+        ),
+        migrations.AlterField(
+            model_name='dockerhubcrawler',
+            name='last_updated',
+            field=models.DateTimeField(blank=True, default=datetime.datetime(2019, 5, 20, 17, 51, 59, 100259, tzinfo=utc), null=True),
+        ),
+    ]

githubCrawler/migrations/0003_auto_20220213_1722.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.0 on 2022-02-13 17:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('githubCrawler', '0002_auto_20220205_1704'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='githubcrawler',
+            name='last_updated',
+        ),
+        migrations.AddField(
+            model_name='githubcrawler',
+            name='last_pushed_number',
+            field=models.IntegerField(null=True),
+        ),
+    ]

githubCrawler/migrations/0004_auto_20220213_1724.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0 on 2022-02-13 17:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('githubCrawler', '0003_auto_20220213_1722'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='api_url',
+            field=models.CharField(blank=True, max_length=1000, null=True),
+        ),
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='last_pushed',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='last_pushed_number',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

githubCrawler/migrations/0005_auto_20220213_1751.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2022-02-13 17:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('githubCrawler', '0004_auto_20220213_1724'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='githubcrawler',
+            name='last_pushed',
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]

githubCrawler/models.py

@@ -1,3 +1,4 @@
+from re import T
 from django.db import models
 
 # Create your models here.
@@ -5,18 +6,18 @@ from django.db import models
 
 class GithubCrawler(models.Model):
     url = models.CharField(max_length=1000,blank = False)
-    api_url = models.CharField(max_length=1000, null= True)
+    api_url = models.CharField(max_length=1000, null= True, blank=True)
-    last_pushed = models.DateTimeField(null= True)
+    last_pushed = models.DateField(null= True,blank=True)
-    last_updated = models.DateTimeField(null= True)
+    last_pushed_number = models.IntegerField(null= True,blank=True)
     def __str__(self):
         return self.url
 
     def save(self, *args, **kwargs):
-        split_string =  self.url.split("/")
+        check_len = len(self.url)
-        self.api_url =  'https://hub.docker.com/v2/repositories/' + split_string[5] +'/' \
+        if self.url[check_len-1] == "/":
-                        + split_string[6] + '/tags/?page=1&page_size=800'
+            self.api_url =  self.url + 'commits/master'
+        else:
+            self.api_url =  self.url + '/commits/master'
         super(GithubCrawler, self).save(*args, **kwargs)
 
 # Research , via Chrome dev tools ;)
-#https://hub.docker.com/r/vectorim/element-web/tags
-#https://hub.docker.com/v2/repositories/vectorim/element-web/tags/?page=1&page_size=800

githubCrawler/urls.py

@@ -1,11 +1,10 @@
 from django.urls import path
+from . import views
 
 
 
 
 urlpatterns = [
-#path('create_routes/<str:string_passed>/<str:start_date>/<str:end_date>/',views.create_routes, name='create_routes'),
+path('github',views.GithubPage, name='GithubPage'),
-#path('check_LoggedIn/',views.logged_in_user, name='logged_in_user'),
-#path('check_Socials/',views.social_media_details, name='social_media_details'),
 
 ]

githubCrawler/views.py

@@ -1,3 +1,57 @@
 from django.shortcuts import render
-
+import requests
+from .models import GithubCrawler
 # Create your views here.
+import requests
+from bs4 import BeautifulSoup
+import datetime
+
+def month_converter(month):
+    months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+    return months.index(month) + 1
+
+
+def GithubPage(request):
+
+    for githubobj in GithubCrawler.objects.all():
+
+        URL = githubobj.api_url
+        page = requests.get(URL)
+        temp = ''
+        Soup = BeautifulSoup(page.text, 'lxml')
+        heading_tags = ["h2"]
+        for tags in Soup.find_all(heading_tags):
+            # print(tags)
+            temp =  tags.text.strip()
+            break # first one is latest no need to loop through all of them
+
+        temp = temp[10:]
+        temp = temp.replace(",", "")
+        temp = temp.split(' ')
+        #print(temp)
+        last_pushed_thing = datetime.datetime(int(temp[3]), int(month_converter(temp[1])), int(temp[2])) # y-m-d
+
+        for div in Soup.findAll('div', attrs={'class':'issue-link js-issue-link'}):
+            print(div.find('a')['href'])
+
+        # finding commit number here and saving in db
+
+        el = Soup.find(class_='issue-link js-issue-link', href=True)
+        temp_result = (el['href'])
+
+        temp_result = temp_result.split('/')
+        last_pushed_number_thing = int(temp_result[len(temp_result)-1])
+
+        if githubobj.last_pushed_number:
+            if githubobj.last_pushed_number < last_pushed_number_thing:
+                print("send email and notify here")
+        
+        githubobj.last_pushed_number = last_pushed_number_thing
+        githubobj.last_pushed = last_pushed_thing
+
+        githubobj.save()
+
+    allobjs = GithubCrawler.objects.all()
+
+    
+    return render(request, 'github.html', context={'githubobjs': allobjs})

requirements.txt

@@ -0,0 +1,32 @@
+backoff==1.11.1
+beautifulsoup4==4.10.0
+certifi==2021.10.8
+charset-normalizer==2.0.10
+click==8.0.3
+coverage==5.5
+gpg==1.16.0
+idna==3.3
+lxml==4.7.1
+noti.py==0.4.0
+numpy==1.22.0
+pandas==1.3.5
+protobuf==3.19.2
+pyonfleet==1.2.1
+PyQt3D==5.15.5
+PyQt5==5.15.6
+PyQt5-sip==12.9.0
+PyQtChart==5.15.5
+PyQtDataVisualization==5.15.5
+PyQtNetworkAuth==5.15.5
+PyQtPurchasing==5.15.5
+PyQtWebEngine==5.15.5
+python-dateutil==2.8.2
+pytz==2021.3
+PyYAML==5.4.1
+ratelimit==2.2.1
+requests==2.27.1
+six==1.16.0
+soupsieve==2.3.1
+TBB==0.2
+tqdm==4.62.3
+urllib3==1.26.8

templates/docker.html

@@ -61,7 +61,7 @@
                 <a class="nav-link" href="#"> Docker </a>
               </li>
               <li class="nav-item">
-                <a class="nav-link" href="">Github</a>
+                <a class="nav-link" href="/github">Github</a>
               </li>
               <li class="nav-item">
                 <a class="nav-link" href="/">Entrance </a>
@@ -96,9 +96,7 @@
                 {% endfor %}
                 </ul>
             </p>
-            <a href="">
+
-              Read More
-            </a>
           </div>
         </div>
         <div class="col-md-6 ">

templates/github.html

@@ -58,13 +58,13 @@
             <ul class="navbar-nav  ">
 
               <li class="nav-item">
-                <a class="nav-link active" href="/docker"> Docker</a>
+                <a class="nav-link" href="/docker"> Docker</a>
+              </li>
+              <li class="nav-item active">
+                <a class="nav-link" href="#"> Github </a>
               </li>
               <li class="nav-item">
-                <a class="nav-link" href="">Github</a>
+                <a class="nav-link" href="/">Entrance  </a>
-              </li>
-              <li class="nav-item">
-                <a class="nav-link" href="/">Entrance <span class="sr-only">(current)</span> </a>
               </li>
             </ul>
           </div>
@@ -82,23 +82,23 @@
           <div class="detail-box">
             <div class="heading_container">
               <h2>
-                About Us
+                Github Crawling Entries
               </h2>
             </div>
             <p>
-              There are many variations of passages of Lorem Ipsum available, but the majority have suffered alteration
+              {% for obj in githubobjs %}
-              in some form, by injected humour, or randomised words which don't look even slightly believable. If you
+              <li>{{ obj.url }}</li>
-              are going to use a passage of Lorem Ipsum, you need to be sure there isn't anything embarrassing hidden in
+              <li>{{ obj.last_pushed }}</li>
-              the middle of text. All
+              <li>{{obj.last_pushed_number}}</li>
+              <br>
+              {% endfor %}
             </p>
-            <a href="">
+
-              Read More
-            </a>
           </div>
         </div>
         <div class="col-md-6 ">
           <div class="img-box">
-            <img src="images/about-img.jpg" alt="">
+            <img src="{% static '/app/images/about-img.jpg' %}" alt="">
           </div>
         </div>
       </div>

templates/home.html

@@ -61,7 +61,7 @@
                 <a class="nav-link" href="/docker"> Docker</a>
               </li>
               <li class="nav-item">
-                <a class="nav-link" href="">Github</a>
+                <a class="nav-link" href="/github">Github</a>
               </li>
               <li class="nav-item active">
                 <a class="nav-link" href="/">Entrance <span class="sr-only">(current)</span> </a>
@@ -120,7 +120,7 @@
               <p>
                 All Githubs Appear Under This section
               </p>
-              <a href="">
+              <a href="/github">
                 Read More
               </a>
             </div>