Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+abstract-and-embeddings.json filter=lfs diff=lfs merge=lfs -text
+scopus/scopus-soc-journals.csv filter=lfs diff=lfs merge=lfs -text

.github/workflows/update_space.yml

@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy

.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

.python-version

@@ -0,0 +1 @@
+3.10

README.md

@@ -1,12 +1,6 @@
 ---
-title: Find Sociology
-emoji: 🦀
-colorFrom: red
-colorTo: gray
+title: Find_Sociology
+app_file: app.py
 sdk: gradio
 sdk_version: 5.28.0
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

abstract-and-embeddings.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc8e995d67cda80ec5cb6e4fdc96adca7b13c66b239f6a1f6e5c83a346dcda2e
+size 267552642

app.py

@@ -0,0 +1,498 @@
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "gradio",
+#     "sentence-transformers",
+#     "pandas",
+#     "numpy",
+#     "scikit-learn",
+#     "torch",
+#     "torchvision",
+#     "torchaudio",
+# ]
+# ///
+
+# 1. Import Libraries
+import gradio as gr
+import pandas as pd
+import numpy as np
+import os
+import json
+import functools
+import re # Import regex for parsing author names/IDs
+from collections import Counter # For counting author occurrences
+
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+import torch # Required by sentence-transformers
+
+# 2. Constants
+MODEL_NAME = 'all-MiniLM-L6-v2'
+SCOPUS_FOLDER = 'scopus'         # Folder containing Scopus CSV files
+DATA_FILE = 'abstract-and-embeddings.json'
+
+# --- Helper Functions ---
+
+# Cache model loading
+@functools.lru_cache(maxsize=None)
+def load_model(model_name=MODEL_NAME):
+    """Loads the Sentence Transformer model."""
+    print(f"Loading Sentence Transformer model: {model_name}...")
+    try:
+        # Check for GPU availability
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        print(f"Using device: {device}")
+        model = SentenceTransformer(model_name, device=device)
+        print("Model loaded successfully.")
+        return model
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        # Fallback to CPU if GPU loading fails unexpectedly
+        try:
+            print("Attempting to load model on CPU...")
+            model = SentenceTransformer(model_name, device='cpu')
+            print("Model loaded successfully on CPU.")
+            return model
+        except Exception as fallback_e:
+            print(f"Fallback CPU loading failed: {fallback_e}")
+            raise
+
+# 3. Data Preprocessing
+def preprocess_data(model, scopus_folder=SCOPUS_FOLDER, data_filepath=DATA_FILE):
+    """
+    Scans the scopus_folder for CSV files, reads them, generates embeddings
+    for new abstracts, and saves to JSON. Avoids adding duplicates based on DOI.
+    Includes 'Author full names'.
+    """
+    processed_data = []
+    processed_dois = set()
+    total_new_entries = 0 # Accumulator for new entries across all files
+
+    # Load existing data if available
+    if os.path.exists(data_filepath):
+        try:
+            with open(data_filepath, 'r', encoding='utf-8') as f:
+                processed_data = json.load(f)
+            # Extract existing DOIs
+            for item in processed_data:
+                if 'doi' in item and item['doi'] is not None: # Check for None DOI
+                    processed_dois.add(str(item['doi']).lower())
+            print(f"Loaded {len(processed_data)} items from {data_filepath}. Found {len(processed_dois)} existing unique DOIs.")
+        except json.JSONDecodeError:
+            print(f"Warning: Could not decode JSON from {data_filepath}. Starting fresh.")
+            processed_data = [] # Reset processed_data
+        except Exception as e:
+            print(f"Warning: Error loading {data_filepath}: {e}. Starting fresh.")
+            processed_data = [] # Reset processed_data
+
+    # Check if Scopus folder exists
+    if not os.path.isdir(scopus_folder):
+        print(f"Error: Scopus folder not found at '{scopus_folder}'")
+        if not processed_data:
+             print("No existing data file found either. Cannot proceed.")
+             return []
+        else:
+             print("Proceeding with existing data from JSON file.")
+             return processed_data
+
+    print(f"Scanning folder '{scopus_folder}' for CSV files...")
+    # Iterate through files in the Scopus folder
+    for filename in os.listdir(scopus_folder):
+        # Check if the file is a CSV
+        if filename.lower().endswith('.csv'):
+            csv_filepath = os.path.join(scopus_folder, filename)
+            print(f"\n--- Processing file: {csv_filepath} ---")
+
+            # Read the Scopus CSV
+            try:
+                # Try reading with default UTF-8, fallback to latin1 if error
+                try:
+                    df = pd.read_csv(csv_filepath, encoding='utf-8')
+                except UnicodeDecodeError:
+                    print(f"UTF-8 decode failed for {filename}, trying latin1...")
+                    df = pd.read_csv(csv_filepath, encoding='latin1')
+                print(f"Successfully read {len(df)} rows from {filename}.")
+            except Exception as e:
+                print(f"Error reading CSV file {filename}: {e}. Skipping this file.")
+                continue # Skip to the next file
+
+            # Ensure required columns exist (Added 'Author full names')
+            required_cols = ['DOI', 'Abstract', 'Title', 'Authors', 'Author full names', 'Year', 'Source title']
+            if not all(col in df.columns for col in required_cols):
+                print(f"Error: CSV file {filename} must contain columns: {required_cols}")
+                missing_cols = [col for col in required_cols if col not in df.columns]
+                print(f"Missing columns: {missing_cols}. Skipping this file.")
+                continue # Skip to the next file
+
+            file_new_entries_count = 0
+            # Prepare abstracts for batch embedding for this file
+            abstracts_to_embed = []
+            corresponding_rows = []
+
+            print(f"Processing rows in {filename}...")
+            # Ensure DOI and Abstract are strings and handle NaNs
+            # Also handle 'Author full names' potentially being float if all NaN
+            df = df.astype({'DOI': str, 'Abstract': str, 'Author full names': str})
+            df.dropna(subset=['DOI', 'Abstract'], inplace=True) # Keep rows even if Author full names is NaN initially
+
+            for index, row in df.iterrows():
+                doi = row['DOI'].lower().strip()
+                abstract = row['Abstract'].strip()
+                author_full_names = row['Author full names'] # Keep as read, handle None later
+
+                # Basic validation - skip if empty after stripping
+                if not doi or not abstract:
+                    continue
+
+                # Check if DOI already processed (from JSON or previous files in this run)
+                if doi in processed_dois:
+                    continue
+
+                # Add abstract and corresponding row index for batch processing
+                abstracts_to_embed.append(abstract)
+                corresponding_rows.append(row)
+                processed_dois.add(doi) # Add DOI here to prevent duplicates
+
+            # Generate embeddings in batches for this file
+            if abstracts_to_embed:
+                print(f"Generating embeddings for {len(abstracts_to_embed)} new abstracts from {filename}...")
+                try:
+                    embeddings = model.encode(abstracts_to_embed, show_progress_bar=True, batch_size=32)
+                    print("Embeddings generated for this batch.")
+
+                    # Add new entries to processed_data
+                    for i, row in enumerate(corresponding_rows):
+                        embedding_list = embeddings[i].tolist()
+                        # Ensure Author full names is stored as string or None
+                        auth_full_names_val = str(row['Author full names']) if pd.notna(row['Author full names']) else None
+
+                        new_entry = {
+                            'doi': str(row['DOI']).strip(),
+                            'title': row['Title'],
+                            'authors': row['Authors'], # Keep the simpler Authors field too
+                            'author_full_names': auth_full_names_val, # Store the detailed field
+                            'year': int(row['Year']) if pd.notna(row['Year']) else None,
+                            'source': row['Source title'],
+                            'abstract': row['Abstract'].strip(),
+                            'embedding': embedding_list
+                        }
+                        processed_data.append(new_entry)
+                        file_new_entries_count += 1
+                    total_new_entries += file_new_entries_count # Add to overall count
+                    print(f"Added {file_new_entries_count} new entries from {filename}.")
+
+                except Exception as e:
+                    print(f"Error during embedding generation or processing for {filename}: {e}")
+                    # Remove DOIs added in this failed batch to allow reprocessing if needed
+                    for row in corresponding_rows:
+                        processed_dois.discard(str(row['DOI']).lower().strip())
+            else:
+                 print(f"No new, unique abstracts found in {filename}.")
+
+    # Save updated data to JSON only if new entries were added across all files
+    if total_new_entries > 0:
+        print(f"\nTotal new entries added: {total_new_entries}. Saving updated data to {data_filepath}...")
+        try:
+            with open(data_filepath, 'w', encoding='utf-8') as f:
+                json.dump(processed_data, f, indent=4)
+            print(f"Successfully saved {len(processed_data)} total items.")
+        except Exception as e:
+            print(f"Error saving data to {data_filepath}: {e}")
+    else:
+        print("\nNo new entries added across all files.")
+
+    return processed_data
+
+# --- Author Parsing Helper ---
+def parse_author_full_names(author_string):
+    """Parses the 'Author full names' string into a list of (name, id) tuples."""
+    authors = []
+    if not author_string or pd.isna(author_string):
+        return authors
+    # Regex to find Name (anything before the last parenthesis) and ID (digits inside last parenthesis)
+    # Handles cases like "Author Name (ID)" and "Author Name" (where ID might be missing)
+    pattern = re.compile(r"^(.*?)\s*\((\d+)\)$")
+    individual_authors = author_string.split(';')
+    for part in individual_authors:
+        part = part.strip()
+        if not part:
+            continue
+        match = pattern.match(part)
+        if match:
+            name = match.group(1).strip()
+            author_id = match.group(2).strip()
+            authors.append({'name': name, 'id': author_id})
+        else:
+            # Handle cases without an ID - use name as ID for counting purposes? Or skip?
+            # For now, let's use the name itself as a pseudo-id if no numeric ID found
+            # This might group authors with same name but different IDs if format varies.
+            # A more robust approach might require cleaner data or skipping entries without IDs.
+            name = part
+            author_id = f"name_{name}" # Create a pseudo-ID based on name
+            authors.append({'name': name, 'id': author_id})
+            # print(f"Warning: Could not parse ID for author '{part}'. Using name as identifier.")
+    return authors
+
+
+# 4. Search Logic (Modified to return Markdown results)
+def find_similar_articles(input_abstract, pos_terms, neg_terms, k, model, all_data):
+    """
+    Finds k similar articles based on abstract embeddings, adjusting for terms.
+    Also identifies authors appearing multiple times in the results.
+    Returns: (repeated_authors_df, results_markdown_string, status_message)
+    """
+    # --- Initial Checks ---
+    empty_df = pd.DataFrame()
+    empty_md = "" # Empty string for Markdown return
+    if not input_abstract:
+        return empty_df, empty_md, "Please enter an abstract."
+    if not all_data:
+         return empty_df, empty_md, "Error: No article data available. Check CSV processing and JSON file."
+
+    # --- Prepare Data and Embeddings ---
+    try:
+        valid_data = [item for item in all_data if 'embedding' in item and isinstance(item['embedding'], list)]
+        if len(valid_data) != len(all_data):
+            print(f"Warning: {len(all_data) - len(valid_data)} items missing valid embeddings. Proceeding with {len(valid_data)} items.")
+        if not valid_data:
+             return empty_df, empty_md, "Error: No items with valid embeddings found."
+
+        stored_embeddings = np.array([item['embedding'] for item in valid_data])
+        article_metadata = valid_data
+    except KeyError:
+         return empty_df, empty_md, "Error: 'embedding' key missing in processed data. Check JSON file structure."
+    except Exception as e:
+         return empty_df, empty_md, f"Error preparing data for search: {e}"
+
+    if stored_embeddings.size == 0:
+         return empty_df, empty_md, "Error: No embeddings found in the data."
+
+    # --- Input Embedding Calculation ---
+    try:
+        input_embedding = model.encode([input_abstract.strip()])[0]
+        if pos_terms and pos_terms.strip():
+            pos_embedding = model.encode([pos_terms.strip()])[0]
+            input_embedding = input_embedding + pos_embedding
+            print("Adjusted embedding with positive terms.")
+        if neg_terms and neg_terms.strip():
+            neg_embedding = model.encode([neg_terms.strip()])[0]
+            input_embedding = input_embedding - neg_embedding
+            print("Adjusted embedding with negative terms.")
+    except Exception as e:
+        return empty_df, empty_md, f"Error generating embeddings for input/terms: {e}"
+
+    # --- Similarity Calculation ---
+    try:
+        similarities = cosine_similarity(input_embedding.reshape(1, -1), stored_embeddings)[0]
+    except Exception as e:
+        return empty_df, empty_md, f"Error calculating similarity: {e}"
+
+    # --- Get Top K Results ---
+    k = min(int(k), len(similarities))
+    if k <= 0:
+         return empty_df, empty_md, "Please select k > 0."
+
+    valid_indices = np.where(~np.isnan(similarities))[0]
+    if len(valid_indices) == 0:
+        return empty_df, empty_md, "Error: Could not compute valid similarities."
+
+    valid_similarities = similarities[valid_indices]
+    sorted_valid_indices_desc = np.argsort(valid_similarities)[::-1]
+    top_k_original_indices = valid_indices[sorted_valid_indices_desc[:k]]
+
+    # --- Format Main Results into Markdown String ---
+    results_markdown_parts = []
+    author_id_list_for_counting = [] # List to hold all author IDs from the results
+    author_id_to_name_map = {} # Map ID to Name
+
+    for rank, i in enumerate(top_k_original_indices):
+        try:
+            article = article_metadata[i]
+            similarity_score = similarities[i]
+            doi_val = article.get('doi')
+            doi_link = f"https://doi.org/{doi_val}" if doi_val else None
+            title = article.get('title', 'N/A')
+            source = article.get('source', 'N/A')
+            abstract = article.get('abstract', 'N/A')
+            author_full_names_str = article.get('author_full_names') # Get the string
+
+            # Parse authors
+            parsed_authors = parse_author_full_names(author_full_names_str)
+            author_names_only = [a['name'] for a in parsed_authors]
+            authors_display_str = "; ".join(author_names_only) if author_names_only else "N/A"
+
+            # Add author IDs for counting
+            for author_info in parsed_authors:
+                author_id_list_for_counting.append(author_info['id'])
+                if author_info['id'] not in author_id_to_name_map:
+                     author_id_to_name_map[author_info['id']] = author_info['name']
+
+            # Format Title (linked if DOI exists)
+            title_md = f"**{title}**"
+            if doi_link:
+                title_md = f"**[{title}]({doi_link})**"
+
+            # Format Abstract using blockquote for indentation
+            # Use HTML for smaller font size within Markdown
+            # FIX: Perform replacements *before* inserting into f-string
+            if abstract:
+                escaped_abstract = abstract.replace("<", "&lt;").replace(">", "&gt;").replace("&", "&amp;")
+                formatted_abstract = escaped_abstract.replace("\n", "<br>> ")
+                abstract_md = f'\n> <span style="font-size: smaller;">{formatted_abstract}</span>'
+            else:
+                abstract_md = ""
+
+
+            # Assemble Markdown for this article
+            article_md = f"### Result {rank + 1} (Similarity: {similarity_score:.4f})\n" \
+                         f"{title_md}\n" \
+                         f"_{source}_\n" \
+                         f"{authors_display_str}\n" \
+                         f"{abstract_md}\n\n" \
+                         f"---" # Separator
+
+            results_markdown_parts.append(article_md)
+
+        except IndexError:
+            print(f"Warning: Index {i} out of bounds for article_metadata (length {len(article_metadata)}). Skipping.")
+            continue
+        except Exception as e:
+            print(f"Warning: Error formatting result for index {i}: {e}. Skipping.")
+            continue
+
+    # Join all parts into a single Markdown string
+    results_markdown_string = "\n".join(results_markdown_parts)
+
+    if not results_markdown_string:
+         return empty_df, empty_md, "No results found matching the criteria."
+
+    # --- Calculate Repeated Authors ---
+    repeated_authors_list = []
+    if author_id_list_for_counting:
+        author_counts = Counter(author_id_list_for_counting)
+        for author_id, count in author_counts.items():
+            if count > 1:
+                author_name = author_id_to_name_map.get(author_id, f"ID: {author_id}") # Get name from map
+                repeated_authors_list.append({
+                    "Author Name": author_name,
+                    "Count": count
+                })
+
+    # Sort repeated authors by count descending
+    repeated_authors_df = pd.DataFrame(repeated_authors_list)
+    if not repeated_authors_df.empty:
+        repeated_authors_df = repeated_authors_df.sort_values(by="Count", ascending=False)
+
+    status_message = f"Found {len(top_k_original_indices)} results. {len(repeated_authors_df)} authors appear more than once."
+
+    return repeated_authors_df, results_markdown_string, status_message
+
+# 5. Gradio Interface Creation (Modified for default k=20)
+def create_gradio_app(processed_data, model):
+    """Creates and returns the Gradio interface."""
+    search_func_with_data = functools.partial(find_similar_articles, model=model, all_data=processed_data)
+
+    with gr.Blocks(theme=gr.themes.Soft()) as demo:
+        gr.Markdown("# Abstract-Based Article Similarity Finder")
+        gr.Markdown(f"Uses Sentence Embeddings (`{MODEL_NAME}`) to find similar articles based on abstracts.")
+        gr.Markdown(f"Data sourced from CSV files in the **`{SCOPUS_FOLDER}`** folder and stored/updated in `{DATA_FILE}`.")
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                input_abstract_box = gr.Textbox(
+                    lines=10,
+                    label="Paste Abstract Here",
+                    placeholder="Enter the abstract text you want to find similar articles for..."
+                )
+                with gr.Row():
+                    pos_terms_box = gr.Textbox(label="Positive Search Terms (Optional)", placeholder="Add terms to boost relevance (e.g., specific methods, concepts)")
+                    neg_terms_box = gr.Textbox(label="Negative Search Terms (Optional)", placeholder="Add terms to decrease relevance (e.g., unrelated topics)")
+
+                # Set default k to 20, ensure max is sufficient
+                max_k = max(20, min(30, len(processed_data) if processed_data else 20)) # Ensure max is at least 20
+                default_k = max(1, min(20, len(processed_data) if processed_data else 1)) # Default to 20 if possible
+                k_slider = gr.Slider(minimum=1, maximum=max_k, step=1, value=default_k, label="Number of Results (k)")
+                submit_button = gr.Button("Find Similar Articles", variant="primary")
+
+            with gr.Column(scale=3):
+                status_textbox = gr.Textbox(label="Status", interactive=False)
+                # DataFrame for repeated authors (remains the same)
+                repeated_authors_dataframe = gr.DataFrame(
+                    label="Authors Appearing Multiple Times in Results",
+                    headers=["Author Name", "Count"],
+                    visible=True
+                )
+                # Changed results display to Markdown
+                results_markdown_display = gr.Markdown(
+                    label="Search Results"
+                )
+
+        # Update outputs for the click event
+        submit_button.click(
+            fn=search_func_with_data,
+            inputs=[input_abstract_box, pos_terms_box, neg_terms_box, k_slider],
+            # Output order: repeated authors DF, results Markdown, status Textbox
+            outputs=[repeated_authors_dataframe, results_markdown_display, status_textbox]
+        )
+
+        # Update outputs for examples, setting k to 20
+        if processed_data and len(processed_data) > 0 and 'abstract' in processed_data[0]:
+             try:
+                 example_abstract_text = processed_data[0].get('abstract', '')
+                 if isinstance(example_abstract_text, str):
+                     example_abstract = example_abstract_text[:500]
+                     if len(example_abstract_text) > 500:
+                         example_abstract += "..."
+                     # Set example k to 20, ensuring it doesn't exceed max_k
+                     example_k = max(1, min(20, len(processed_data) if processed_data else 1))
+                     example_k = min(example_k, max_k) # Ensure example k doesn't exceed slider max
+
+                     gr.Examples(
+                         # Use example_k for the example
+                         examples=[[example_abstract, "", "", example_k]],
+                         inputs=[input_abstract_box, pos_terms_box, neg_terms_box, k_slider],
+                         # Update outputs for examples as well
+                         outputs=[repeated_authors_dataframe, results_markdown_display, status_textbox],
+                         fn=search_func_with_data,
+                         cache_examples=False
+                     )
+                 else:
+                     print("Warning: Could not create example because the first abstract is not a string.")
+             except Exception as e:
+                 print(f"Could not create example: {e}")
+
+    return demo
+
+
+# 6. Main Execution Block
+if __name__ == "__main__":
+    # --- Create Scopus Folder if it doesn't exist (for testing) ---
+    if not os.path.exists(SCOPUS_FOLDER):
+        print(f"Creating folder '{SCOPUS_FOLDER}' as it does not exist.")
+        os.makedirs(SCOPUS_FOLDER)
+        if os.path.exists('scopus-24.csv'): # Check for the specific file you uploaded
+             try:
+                 import shutil
+                 shutil.copy('scopus-24.csv', os.path.join(SCOPUS_FOLDER, 'scopus-24.csv'))
+                 print(f"Copied 'scopus-24.csv' into '{SCOPUS_FOLDER}' for testing.")
+             except Exception as e:
+                 print(f"Could not copy 'scopus-24.csv': {e}")
+        else:
+             print(f"Place your Scopus CSV files inside the '{SCOPUS_FOLDER}' directory.")
+    # --- End of folder creation ---
+
+
+    # Load the model once
+    sbert_model = load_model(MODEL_NAME)
+
+    # Preprocess data (load existing, check CSVs in folder for new)
+    print("--- Starting Data Preprocessing ---")
+    all_processed_data = preprocess_data(sbert_model)
+    print("--- Data Preprocessing Finished ---")
+    if not all_processed_data:
+        print(f"Warning: No data loaded or processed. Ensure CSV files exist in '{SCOPUS_FOLDER}' and are valid.")
+
+    # Create and launch the Gradio app
+    app = create_gradio_app(all_processed_data, sbert_model)
+    print("Launching Gradio app...")
+    app.launch()

pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "scopus-ab"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []

requirements.txt

@@ -0,0 +1,8 @@
+gradio
+sentence-transformers
+pandas
+numpy
+scikit-learn
+torch
+torchvision
+torchaudio

scopus/scopus-soc-journals.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a80d8e268a4a6a2a113f7d5f2e4be6192af0ec84f9e32edb25a371c32be2c615
+size 31758963

uv.lock

@@ -0,0 +1,7 @@
+version = 1
+requires-python = ">=3.10"
+
+[[package]]
+name = "scopus-ab"
+version = "0.1.0"
+source = { virtual = "." }