Created initial data project structure - LJD 7/21/26

.gitignore

@@ -190,5 +190,14 @@ cython_debug/
 #  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
 #  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
 #  refer to https://docs.cursor.com/context/ignore-files
-.cursorignore
-.cursorindexingignore
+# .cursorignore
+# .cursorindexingignoredata/movies-archive/
+
+# Ignore all project data
+data/
+
+# macOS
+.DS_Store
+
+# VS Code
+.vscode/

src/books.py

@@ -0,0 +1,191 @@
+import csv
+import json
+import random
+import re
+
+from pathlib import Path
+
+csv_file = Path("data/pg_catalog.csv")
+json_file = Path("data/booksout.json")
+used_isbns = set()
+
+def generate_isbn():  #generate a unique 13 digit isbn-like number
+
+    while True:
+        isbn = str(random.randint(1000000000000, 9999999999999))
+
+        if isbn not in used_isbns:
+            used_isbns.add(isbn)
+            return isbn
+
+def read_books(csv_file):
+    with open(csv_file, 'r', encoding="utf-8") as file:
+        reader = csv.DictReader(file)
+        books = []
+
+        for row in reader:
+            books.append(row) 
+
+    return books 
+
+def get_genre(bookshelves_field):
+    if not bookshelves_field:
+        return "Not Available"
+
+    bookshelves = bookshelves_field.split(";")
+
+    # genres = []
+
+    for shelf in bookshelves:
+        shelf = shelf.strip()
+
+        if shelf.startswith("Browsing:"):
+            shelf = shelf.replace("Browsing:", "", 1)
+            shelf = shelf.strip()
+
+        if shelf:
+            return shelf
+
+    return "Not Available"
+
+def clean_books(books):
+    """1.  Add an isbn to each book/row"""
+
+    cleaned_books = []
+    for book in books:
+        parsed_authors = clean_authors(book.get("Authors", ""))
+        parsed_subjects = clean_subjects(book.get("Subjects", ""))
+
+        cleaned_book = {  
+            "recordID": book.get("Text#", "").strip(),         
+            "title": book.get("Title", "").strip(),
+            "authors": [
+                author["name"]
+                for author in parsed_authors
+            ],
+            "author_lifespan": [
+                author["lifespan"]
+                for author in parsed_authors
+                if author["lifespan"]
+            ],
+            "isbn": generate_isbn(),
+            "numberOfPages": 0,
+
+            "genre": get_genre(book.get("Bookshelves", "")),
+            "subjects": parsed_subjects,
+            "locc": book.get("LoCC", "").strip()   
+        }
+
+        cleaned_books.append(cleaned_book)
+
+    return cleaned_books
+
+def clean_authors(author_field):
+    if not author_field:
+        return[]
+
+    authors = author_field.split(";")
+    cleaned_authors = []
+
+    for author in authors:
+        author = author.strip()
+
+        lifespan_match = re.search(
+            r"\b\d{3,4}\??(?:\s+BCE)?-\d{3,4}\??(?:\s+BCE)?\b",
+            author
+        )
+        if lifespan_match:
+            lifespan = lifespan_match.group()
+
+            name = author.replace(lifespan, "")
+            name = name.strip(" ,")
+        else:
+            lifespan = ""
+            name = author
+
+        cleaned_authors.append({
+            "name": name,
+            "lifespan": lifespan
+        })
+
+    return cleaned_authors
+
+def clean_subjects(subject_field):
+    if not subject_field:
+        return []
+
+    subjects = subject_field.split(";")
+    cleaned_subjects = []
+
+    for subject in subjects:
+        subject = subject.strip()
+
+        if subject:
+            cleaned_subjects.append(subject)
+
+    return cleaned_subjects
+
+def validate_books(cleaned_books):
+    valid_books = []
+    missing_title = 0
+    missing_authors = 0
+    invalid_isbn = 0
+    #record_number = 1
+
+    for book in cleaned_books:
+
+        title = book.get("title", "").strip()
+        authors = book.get("authors", [])
+        isbn = book.get("isbn", "").strip()
+
+        if not title:
+            missing_title += 1
+            continue
+
+        if not authors:
+            missing_authors += 1
+            continue
+
+        if len(isbn) != 13:
+            invalid_isbn += 1
+            continue
+
+        valid_books.append(book)
+
+    print("Missing title:", missing_title)
+    print("Missing authors:", missing_authors)
+    print("Invalid ISBN:", invalid_isbn)
+    print("Books validated:", len(valid_books))
+
+    return valid_books
+
+def write_json(valid_books, json_file):
+    with open(json_file, "w", encoding="utf-8") as file:
+        json.dump(valid_books, file, indent=4)
+
+    return json_file
+
+
+if __name__ == "__main__":
+    books = read_books(csv_file)
+    cleaned_books = clean_books(books)
+    valid_books = validate_books(cleaned_books)
+    books_to_process = valid_books
+    books_to_write = valid_books
+    # books_to_process = valid_books[:50]
+    # books_to_write = valid_books[:50]
+
+
+    print("Books read: ", len(books))
+    print("Books cleaned: ", len(cleaned_books))
+    print("Books validated: ", len(valid_books))
+    print("Books being written to JSON file: ", len(books_to_write))
+
+
+    # write_json(valid_books[:50], json_file)
+    write_json(valid_books, json_file)
+
+    #this is so only the first 5 rows are returned for testing purposes
+    # for book in books[:5]:
+    #     print(book)
+

src/dvd.py

@@ -0,0 +1,144 @@
+import csv
+import json
+import ast
+# import pandas as pd
+import requests
+import os
+
+API_KEY = os.getenv("OMDB_API_KEY")
+rating_cache = {}
+
+from pathlib import Path
+
+credit_file = Path("data/movies-archive/credits.csv")
+movies_file = Path("data/movies-archive/movies_metadata.csv")
+json_file = Path("data/dvdout.json")
+
+
+def get_movie_rating(imdb_id, api_key):
+    if not imdb_id:
+        return "Not Rated"
+
+    if imdb_id in rating_cache:
+        return rating_cache[imdb_id]
+
+    url = "https://www.omdbapi.com/"
+    parameters = {
+        "apikey": api_key,
+        "i": imdb_id
+    }
+    try:
+        response = requests.get(url, params=parameters, timeout=10)
+        response.raise_for_status()
+        movie_data = response.json()
+        rating = movie_data.get("Rated", "NR")
+    except requests.RequestException:
+        rating = "NR"
+
+    rating_cache[imdb_id] = rating
+    return rating
+
+
+def read_movies_metadata(movies_file):
+    movies = []
+    with open(movies_file, "r", encoding="utf-8") as file:
+        reader = csv.DictReader(file)
+
+        for row in reader:
+            movies.append(row)
+
+    return movies
+
+def clean_movies(movies):
+    pass
+
+def read_credits(credit_file):
+    credits = []
+    with open(credit_file, "r", encoding="utf-8") as file:
+        reader = csv.DictReader(file)
+
+        for row in reader:
+            credits.append(row)
+
+    return credits
+
+def get_director(crew_string):
+
+    try:
+        crew_list = ast.literal_eval(crew_string)
+    except (ValueError, SyntaxError):
+        return "Unknown"
+
+    for crew_member in crew_list:
+        if crew_member.get("job") == "Director":
+            return crew_member.get("name")
+
+    return "Unknown"
+
+def get_genres(genres_string):
+    try:
+        genres_list = ast.literal_eval(genres_string)
+
+        genre_names = []
+
+        for genre in genres_list:
+            genre_names.append(genre["name"])
+
+        return ", ".join(genre_names)
+
+    except (ValueError, SyntaxError):
+        return "Unknown"
+
+def merge_files(movies, credits_data):
+    credit_lookup = {}
+
+    # get the director name
+    for credit in credits_data:
+        credit_lookup[credit["id"]] = get_director(credit["crew"])
+
+    merged_movies = []
+    record_number = 1
+
+    for movie in movies:
+        movie_id = movie.get("id", "")
+        imdb_id = movie.get("imdb_id", "")
+
+        director = credit_lookup.get(movie_id, "Unknown")
+        #rating = get_movie_rating(imdb_id, API_KEY)
+        rating = "NOT YET LOADED FOR TEST"    #TESTING PURPOSES ONLY HARDCODING RATING
+
+        merged_movie = {
+            "recordID": f"{record_number:06d}",
+            "title": movie.get("title", ""),
+            "director": director,
+            "duration": movie.get("runtime", ""),
+            "rating": rating,
+            "genre": get_genres(movie.get("genres", "[]"))
+        }
+
+        merged_movies.append(merged_movie)
+        record_number += 1
+
+    return merged_movies
+
+def write_json(movies, json_file):
+    with open(json_file, "w", encoding="utf-8") as file:
+        json.dump(movies, file, indent=4)
+
+    return json_file
+
+
+if __name__ == "__main__":
+
+    # print(get_movie_rating("tt0114709", API_KEY))
+
+    movies = read_movies_metadata(movies_file)
+    movies_to_process = movies[:50]
+
+    credits_data = read_credits(credit_file)
+
+    merged_movies = merge_files(
+        movies_to_process, credits_data
+    )
+
+    write_json(merged_movies, json_file)