Updated code, requirements and README

.gitignore

@@ -5,15 +5,13 @@ __pycache__/
 __init__.py
 local_settings.py
 
-.env
 db.sqlite3
 
 /static/
 /media/
 /main/log
 /main/migrations
 
-env/
 
 node_modules/
 package-lock.json
@@ -22,3 +20,7 @@ tags
 gunicorn/gunicorn.conf.py
 gunicorn/gunicorn.env
 recom_data.csv
+
+data/
+venv/
+staticfiles/

README.md

@@ -4,18 +4,22 @@ NISER Archive
 #### How to run a local instance?
 
  * Sit tight. ~~If migrations fail, delete everything (the repo, the database, etc.) and start over.~~ (I have learnt this the hard way: DO NOT TRACK MIGRATIONS FILES, and everything works fine.)
- * Install: `python`, `postgresql`
+ * Install:
+    * `python` : Install python 3.9.25 or lower preferably using pyenv(or the more modern uv) as django 2.2.28 does not support python 3.10 and above
+    * `postgresql` : Last checked with postgresql18
  * Clone the repo.
- * Create a [virtual environment](https://docs.python.org/3/tutorial/venv.html) and run `pip install -r requirements.txt`.
- * You'll have to provide [`/arc/local_settings.py`](https://pastebin.com/S9yV4yj5).
- * ~~You'll also have to fix some of the absolute paths in `/arc/settings.py` because~~ I am lazy.
- * `cd` to `/main/static/main` and
+ * Create a [virtual environment](https://docs.python.org/3/tutorial/venv.html) using the python version you installed and run `pip install -r requirements.txt`.
+ * You'll have to provide `/arc/local_settings.py`(look at the example file attached).
+ * Install nodejs and npm systemwide(if not present already) and then 
+ `cd` to `/main/static/main` and
  `npm install jquery popper.js bootstrap katex showdown open-iconic`.
+ * Make a new database in postgresql and register the database and db account credentials in local_settings.py
  * `cd` to the cloned repo (while you're still in virtual env) and run: `python manage.py collectstatic`, `python manage.py makemigrations main`, `python manage.py migrate`
  * Start the server: `python manage.py runserver` or you can run an apache
    server, thats how the deployed server is running presently. Configuring an
    apache server is very machine-specific. Google how to do it on your
    machine.)
+ * For verification mails, update the 'dmn' variable in views.py to the current ip/domain at which you are hosting the site.
  * Please let me know if you're unable to run it on your machine.
 
  #### TODO:

add_courses_and_items.py

@@ -0,0 +1,237 @@
+# Django script to add all courses and items from data/SDG_NISER/*/*
+# Run this in `python manage.py shell` or as a Django management command
+
+import os
+import re
+import sys
+import django
+
+# 1. Point Django to your settings module ('arc.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'arc.settings')
+
+# 2. Add the project root to the system path if your script is in a subfolder
+# (Adjust this path if your script lives somewhere else!)
+sys.path.append('/home/ccarchive/archive/arc')
+
+# 3. Initialize Django
+django.setup()
+
+from django.core.files import File
+from django.utils import timezone
+from main.models import School, Course, Itr, Item
+from authtools.models import User
+
+# Set operator user
+op = User.objects.get(email="sandipan.samanta@niser.ac.in")
+
+# Path to the data root
+base_dir = "/home/ccarchive/archive/arc/staticfiles/SDG_NISER"
+
+# print("Checking and initializing Schools...")
+
+# SCHOOLS_DATA = {
+#     "SBS": "School of Biological Sciences",
+#     "SCS": "School of Chemical Sciences",
+#     "SMS": "School of Mathematical Sciences",
+#     "SPS": "School of Physical Sciences",
+#     "SEPS": "School of Earth and Planetary Sciences",
+#     "SCoS": "School of Computer Sciences",
+#     "SHSS": "School of Humanities and Social Sciences",
+#     "CMRP": "Centre for Medical and Radiation Physics"
+# }
+
+# for abbr, full_name in SCHOOLS_DATA.items():
+#     # If your School model requires the 'op' or 'appr' fields like your Course model does, 
+#     # add them to the defaults dictionary below.
+#     school, created = School.objects.get_or_create(
+#         abbr=abbr,
+#         defaults={"name": full_name}
+#     )
+#     if created:
+#         print(f"  + Created new school: {abbr} - {full_name}")
+
+# Regex for <Course_Code>_<course_name_in_snake_case>
+course_pattern = re.compile(r"^([A-Za-z0-9]+)_(.+)$")
+
+# Fixed regex pattern for files
+# Matches: coursecode_itemname_year.pdf, coursecode_itemname_year_.pdf, etc.
+# Examples: b202_endsem_2016.pdf, b202_endsem_2021_.pdf, b202_quiz_1_2021.pdf
+# file_pattern = re.compile(r"^([a-z0-9]+)_(.+?)_(\d{4})_?\.pdf$", re.IGNORECASE)
+# A list of patterns to try, from most specific to least specific
+FILE_PATTERNS = [
+    # 1. Standard (Code, Item, Year): b202_endsem_2021.pdf, b202-quiz 1-2021_sol.pdf
+    re.compile(r"^([a-z0-9]+)[_\-\s]+(.+?)[_\-\s]+(\d{4}).*\.pdf$", re.IGNORECASE),
+
+    # 2. Year at the front (Year, Code, Item): 2021_b202_endsem.pdf
+    re.compile(r"^(\d{4})[_\-\s]+([a-z0-9]+)[_\-\s]+(.+)\.pdf$", re.IGNORECASE),
+
+    # 3. No Year included (Code, Item): b202_endsem.pdf
+    re.compile(r"^([a-z0-9]+)[_\-\s]+(.+)\.pdf$", re.IGNORECASE)
+]
+
+# ==========================================
+# File Tracking Statistics
+# ==========================================
+stats = {
+    "total_seen": 0,
+    "added": 0,
+    "skipped_existing": 0,
+    "skipped_regex": 0,
+    "skipped_non_pdf": 0,
+    "skipped_mismatch": 0,
+    "errors": 0
+}
+
+print("\nStarting course and item import...")
+
+for school in School.objects.all():
+    school_dir = os.path.join(base_dir, school.abbr)
+    if not os.path.isdir(school_dir):
+        print(f"Skipping {school.abbr}: directory not found")
+        continue
+
+    print(f"\nProcessing school: {school.abbr}")
+
+    for course_dir in os.listdir(school_dir):
+        course_match = course_pattern.match(course_dir)
+        if not course_match:
+            print(f"  Skipping directory {course_dir}: doesn't match course pattern")
+            continue
+
+        code = course_match.group(1).upper()  # Convert to uppercase for consistency
+        if len(code) > 6:
+            print(f"  Skipping {course_dir}: code '{code}' too long for Course.code field")
+            continue
+
+        name_snake = course_match.group(2)
+        name = name_snake.replace("_", " ").title()  # Better capitalization
+
+        # Get or create course
+        course, created = Course.objects.get_or_create(
+            code=code,
+            school=school,
+            defaults={"op": op, "name": name, "appr": True}
+        )
+
+        if created:
+            print(f"  Created course: {course}")
+        else:
+            print(f"  Found existing course: {course}")
+
+        full_course_dir = os.path.join(school_dir, course_dir)
+
+        # Pass 1: Collect all years from files and create iterations
+        years_found = set()
+        valid_files = []
+
+        for fname in os.listdir(full_course_dir):
+            stats["total_seen"] += 1  # Count every file we look at
+
+            if not fname.lower().endswith('.pdf'):
+                print(f"    Skipping non-PDF: {fname}")
+                stats["skipped_non_pdf"] += 1
+                continue
+
+            file_code = None
+            item_name_raw = None
+            year = None
+
+            # (Assuming you are using the updated FILE_PATTERNS list from earlier)
+            match = FILE_PATTERNS[0].match(fname)
+            if match:
+                file_code, item_name_raw, year = match.groups()
+            elif FILE_PATTERNS[1].match(fname):
+                match = FILE_PATTERNS[1].match(fname)
+                year, file_code, item_name_raw = match.groups()
+            elif FILE_PATTERNS[2].match(fname):
+                match = FILE_PATTERNS[2].match(fname)
+                file_code, item_name_raw = match.groups()
+                year = "0000" 
+
+            if not file_code:
+                print(f"    ⚠️ UNMATCHED FORMAT, SKIPPING: {fname}")
+                stats["skipped_regex"] += 1
+                continue
+
+            file_code = file_code.upper()
+
+            if file_code != code:
+                print(f"    Skipped (code mismatch): {file_code} != {code} for {fname}")
+                stats["skipped_mismatch"] += 1
+                continue
+
+            years_found.add(year)
+            valid_files.append((fname, file_code, item_name_raw, year))
+
+        print(f"    Found years: {sorted(years_found)}")
+
+        # Create all iterations for this course
+        for year in years_found:
+            itr, created = Itr.objects.get_or_create(
+                course=course,
+                year=year,
+                defaults={
+                    "op": op, 
+                    "appr": True,
+                    "sem": "FA",  # Default to Fall semester
+                    "inst": "Unknown"  # Default instructor
+                }
+            )
+            if created:
+                print(f"    Created iteration: {itr}")
+
+        # Pass 2: Add items to each iteration
+        for fname, file_code, item_name_raw, year in valid_files:
+            item_name = item_name_raw.replace("_", " ").replace("-", " ").strip()
+            item_name = " ".join(word.capitalize() for word in item_name.split())
+
+            itr = Itr.objects.get(course=course, year=year)
+
+            # Check if item already exists
+            if Item.objects.filter(itr=itr, name=item_name).exists():
+                print(f"    Skipped existing item: {item_name} in {itr}")
+                stats["skipped_existing"] += 1
+                continue
+
+            # Create the item
+            src_path = os.path.join(full_course_dir, fname)
+            try:
+                with open(src_path, "rb") as f:
+                    django_file = File(f, name=fname)
+                    item = Item(
+                        op=op,
+                        itr=itr,
+                        name=item_name,
+                        appr=True,
+                        time=timezone.now(),
+                        desc=f"Imported from {fname}"
+                    )
+                    item.fl.save(fname, django_file, save=False)
+                    item.save()
+
+                print(f"    ✓ Added item: {item_name} to {itr}")
+                stats["added"] += 1  # Track success!
+
+            except Exception as e:
+                print(f"    ✗ Error adding {fname}: {str(e)}")
+                stats["errors"] += 1  # Track errors
+
+print("\n✅ Import completed!")
+
+print(f"\n{'='*50}")
+print(" 📊 IMPORT EXECUTION SUMMARY")
+print(f"{'='*50}")
+print(f"Total Files Scanned in Folders:  {stats['total_seen']}")
+print(f"  ✓ Successfully Added to DB:    {stats['added']}")
+print(f"  ⏭ Skipped (Already in DB):     {stats['skipped_existing']}")
+print(f"  ⏭ Skipped (Not a PDF file):    {stats['skipped_non_pdf']}")
+print(f"  ⚠️ Skipped (Unmatched Regex):   {stats['skipped_regex']}")
+print(f"  ⚠️ Skipped (Folder Mismatch):   {stats['skipped_mismatch']}")
+print(f"  ❌ Errors during upload:       {stats['errors']}")
+print(f"{'-'*50}")
+print(" 🗄️ CURRENT DATABASE TOTALS")
+print(f"{'-'*50}")
+print(f"Total Courses:    {Course.objects.count()}")
+print(f"Total Iterations: {Itr.objects.count()}")
+print(f"Total Items:      {Item.objects.count()}")
+print(f"{'='*50}\n")

arc/settings.py

@@ -111,3 +111,14 @@
 }
 
 DEFAULT_FROM_EMAIL = 'NISER Archive'
+
+# Ensure these are locked to your project's BASE_DIR
+STATIC_URL = '/arc/static/'
+STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
+
+STATICFILES_DIRS = [
+   "/media/data/Anurag/Downloads/archive/arc/main/static",
+]
+
+MEDIA_URL = '/arc/media/'
+MEDIA_ROOT = os.path.join(BASE_DIR, 'media')

arc/urls.py

@@ -13,12 +13,18 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.contrib import admin
 from django.urls import include, path
+from django.conf import settings
+from django.conf.urls.static import static
 
 urlpatterns = [
     path('arc/admin/', admin.site.urls),
     path('arc/', include('main.urls')),
 ]
 
 handler404 = 'main.views.error404'
+
+# Serve media files in development
+urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)