app.py

import gradio as gr
import psutil
import subprocess
import time

#placeholders for api use
def generate_response_by_api(user_message):
    FinalOutput = ""
    #return FinalOutput
    pass

def custom_generate_response_by_api(cust_user_message, prompt_index, prompts_list):
    prompt, ending = prompts_list[prompt_index]  # Unpack the prompt and its ending from the provided list
    cust_user_message = f"{prompt}\n\n{cust_user_message}\n\n{ending}"
    #return generate_response(cust_user_message)
    pass

#-----------------------------------------------------------------------------------------------------------------------

#Local gguf model using llama.cpp
def generate_response(user_message): #generate_response_token_by_token
    cmd = [
        "/app/llama.cpp/llama-cli",  # Path to the executable  #https://github.com/ggerganov/llama.cpp/pull/7809
        "-m", "/app/llama.cpp/models/stablelm-2-zephyr-1_6b-Q4_0.gguf",
        "-p", user_message,
        "-n", "400",
        "-e"
    ]

    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1)
    process_monitor = psutil.Process(process.pid)


    start_time = time.time()
    monitor_start_time = time.time()
    alltokens = ""
    token_buffer = ''
    tokencount = 0
    try:
        while True:
            # Read one character at a time
            char = process.stdout.read(1)
            if char == '' and process.poll() is not None:
                break
            if char != '':
                token_buffer += char
                if char == ' ' or char == '\n':  # Token delimiters
                    elapsed_time = time.time() - start_time  # Calculate elapsed time
                    alltokens += token_buffer
                    tokencount += 1
                    yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds | Tokens: { tokencount }]"
                    token_buffer = ''  # Reset token buffer
            # Log resource usage every minute
            if time.time() - monitor_start_time > 60:
                cpu_usage = process_monitor.cpu_percent()
                memory_usage = process_monitor.memory_info().rss  # in bytes 
                print(f"Subprocess CPU Usage: {cpu_usage}%, Memory Usage: {memory_usage / 1024 ** 2} MB")
                monitor_start_time = time.time()  # Reset the timer
    
        # Yield the last token if there is any
        if token_buffer:
            elapsed_time = time.time() - start_time  # Calculate elapsed time
            alltokens += token_buffer
            yield f"{alltokens} \n\n [Inference time: {elapsed_time:.2f} seconds | Average Tokens per second: { round(tokencount / elapsed_time, 2) }]"

    finally:
        try:
            # Wait for the process to complete, with a timeout
            process.wait(timeout=60)  # Timeout in seconds
        except subprocess.TimeoutExpired:
            print("Process didn't complete within the timeout. Killing it.")
            process.kill()
            process.wait()  # Ensure proper cleanup
        # Wait for the subprocess to finish if it hasn't already
        process.stdout.close()
        process.stderr.close()

        # Check for any errors
        if process.returncode != 0:
            error_message = process.stderr.read()
            print(f"Error: {error_message}")
            
def custom_generate_response(cust_user_message, prompt_index, category):
    """
    Generates a custom response based on the user message, the selected prompt,
    and the provided list of prompts, including a custom ending specific to the prompt.

    Parameters:
    - cust_user_message: The message input from the user.
    - prompt_index: The index of the custom prompt to use.
    - category: The key of list of prompts to use for generating the response.
    """
    prompts_list = Allprompts[category]  # Retrieve the correct list of prompts based on the category
    prompt, ending = prompts_list[prompt_index]  # Unpack the prompt and its ending
    cust_user_message = f"{prompt}\n\n{cust_user_message}\n\n{ending}"
    yield from generate_response(cust_user_message)

Allprompts = {
    "Expansive Problem solving": [
        ("My problem to solve is", "- please make 10 sub problems have to solve from this:"),
        ("My process to solve is", "- please make 10 sub processes have to solve from this:"),
        ("My goal to solve is", "- please make 10 sub goals have to solve from this:"),
        ("My task to solve is", "- please make 10 sub tasks have to solve from this:"),
        ("My phase to solve is", "- please make 10 sub phases have to solve from this:"),
        ("My component to solve is", "- please make 10 sub components have to solve from this:"),
        ("My element to solve is", "- please make 10 sub elements have to solve from this:"),
        ("A brief description of my current situation:", "- please list the most important task to pay attention to:"),
        ("A brief description of my current situation to analyse:", "- please conduct a situational analysis:"),
        ("A brief description of my current situation to decompose:", "- please conduct a problem decomposition:"),
    ],
    "Random Career Based": [
        ("Match the text to any function - Doctors - diagnose_illness(patient_symptoms), prescribe_medication(illness), perform_surgery(patient, surgery_type), conduct_medical_research(research_topic), provide_preventive_care(patient)", "Areas that match:"),
        ("Match the text to any function - Nurses - administer_medication(patient, medication), assist_in_surgeries(surgery_type), monitor_patient_vitals(patient), educate_patients_on_health_management(health_topic), provide_emotional_support(patient)", "Areas that match:"),
        ("Match the text to any function - Pharmacists - dispense_prescriptions(prescription), consult_on_medication_use(medication, patient), conduct_health_screenings(screening_type), manage_pharmacy_operations(), review_medication_therapy(patient, medication)", "Areas that match:"),
        ("Match the text to any function - Dentists - conduct_oral_examinations(patient), perform_dental_procedures(patient, procedure_type), diagnose_oral_diseases(disease_symptoms), prescribe_treatments(for_disease), promote_oral_health(education_topics)", "Areas that match:"),
        ("Match the text to any function - Doctors - diagnose_illness(patient_symptoms), prescribe_medication(illness), perform_surgery(patient, surgery_type), conduct_medical_research(research_topic), provide_preventive_care(patient)", "Areas that match:"),
        ("Match the text to any function - Nurses - administer_medication(patient, medication), assist_in_surgeries(surgery_type), monitor_patient_vitals(patient), educate_patients_on_health_management(health_topic), provide_emotional_support(patient)", "Areas that match:"),
        ("Match the text to any function - Pharmacists - dispense_prescriptions(prescription), consult_on_medication_use(medication, patient), conduct_health_screenings(screening_type), manage_pharmacy_operations(), review_medication_therapy(patient, medication)", "Areas that match:"),
        ("Match the text to any function - Dentists - conduct_oral_examinations(patient), perform_dental_procedures(patient, procedure_type), diagnose_oral_diseases(disease_symptoms), prescribe_treatments(for_disease), promote_oral_health(education_topics)", "Areas that match:"),
        ("Match the text to any function - Psychologists - assess_patient_mental_health(patient_history), develop_treatment_plans(patient_needs), conduct_therapy_sessions(patient), perform_psychological_testing(tests), research_behavioral_patterns(research_topic)", "Areas that match:"),
        ("Match the text to any function - Lawyers - advise_clients_on_legal_matters(client_case), represent_clients_in_court(case_details), draft_legal_documents(document_type), negotiate_settlements(opposing_party), conduct_legal_research(research_topic)", "Areas that match:"),
        ("Match the text to any function - Paralegals - assist_with_case_preparation(case_details), conduct_legal_research(research_topic), draft_legal_documents(document_type), organize_case_files(client_case), facilitate_client_communication(communication_details)", "Areas that match:"),
        ("Match the text to any function - Engineers - design_projects(project_requirements), develop_prototypes(prototype_specifications), test_solutions(test_conditions), analyze_data(data_sets), oversee_project_management(project_details)", "Areas that match:"),
        ("Match the text to any function - IT Professionals - develop_software(software_requirements), manage_network_security(security_measures), troubleshoot_technical_issues(issue_details), implement_data_management_solutions(data_requirements), stay_updated_with_technology_trends(trend_topic)", "Areas that match:"),
        ("Match the text to any function - Accountants - prepare_financial_statements(company_details), conduct_audits(audit_plan), manage_budgets(budget_details), provide_tax_advice(tax_laws), oversee_payroll_services(employee_details)", "Areas that match:"),
        ("Match the text to any function - Financial Advisors - assess_client_financial_situations(client_profile), develop_investment_strategies(investment_goals), provide_retirement_planning(advice), offer_tax_planning(guidance), monitor_client_portfolios(portfolio_details)", "Areas that match:"),
        ("Match the text to any function - Teachers - develop_lesson_plans(curriculum_requirements), facilitate_classroom_instruction(lesson_topic), assess_student_performance(test_scores), provide_individual_student_support(student_needs), engage_in_professional_development(development_opportunity)", "Areas that match:"),
        ("Match the text to any function - Real Estate Agents - evaluate_property_values(property_details), facilitate_property_viewings(client_schedule), negotiate_sales_contracts(contract_terms), advise_clients_on_market_conditions(market_analysis), manage_property_listings(listing_details)", "Areas that match:"),
        ("Match the text to any function - Electricians - install_electrical_systems(building_plan), troubleshoot_electrical_problems(issue_description), conduct_maintenance_checks(maintenance_schedule), ensure_compliance_with_safety_regulations(regulation_details), upgrade_existing_electrical_systems(system_requirements)", "Areas that match:"),
        ("Match the text to any function - Plumbers - install_plumbing_systems(building_design), repair_plumbing_issues(issue_description), conduct_system_maintenance(maintenance_plan), inspect_plumbing_systems_for_compliance(compliance_standards), advise_on_water_conservation(measures)", "Areas that match:"),
        ("Match the text to any function - Pilots - operate_aircraft(flight_plan), conduct_pre-flight_inspections(inspection_checklist), navigate_routes(route_details), ensure_passenger_safety(safety_procedures), communicate_with_air_traffic_control(ATC_instructions)", "Areas that match:"),
        ("Match the text to any function - Seafarers - navigate_vessels(vessel_details), maintain_vessel_safety(safety_standards), conduct_cargo_operations(cargo_details), oversee_vessel_maintenance(maintenance_schedule), ensure_regulatory_compliance(compliance_requirements)", "Areas that match:"),
        ("Match the text to any function - Scientists - conduct_research(research_question), publish_findings(findings_details), develop_experimental_designs(experiment_requirements), analyze_research_data(data_sets), collaborate_with_other_scientists(collaboration_details)", "Areas that match:"),
    ],
    "Random Custom Prompts" : [
        ("Write a Class Diagram based on the following text:", "Class Diagram:"),
        ("Write a Pydot code based on the following text:", "Pydot Code:"),
        ("Describe what a standard happy scene in any movie would be planned in great detail, based on the following text:", "Scene Details"),
        ("Explain a teardown of the product mentioned in the following text:", "Teardown Details:"),
        ("Explain the manufacturing of the product mentioned in the following text:", "Manufacturing Details:"),
        ("Explain the marketing considerations of the product mentioned in the following text:", "Considerations:"),
        ("Explain the target users considerations of the product mentioned in the following text:", "Target Users Considerations:"),
    ],
    "Business Prompts" : [
        ("Suggest Product ideas just based off the following text:", "Products:"),
        ("Write an outline for a business plan for: " , ""),
        ("Write an example of a detailed report for a Executive Summary for " , "Executive Summary:"),
        ("Write an example of a detailed report for a Company Description for " , "Company Description:"),
        ("Write an example of a detailed report for a Market Analysis for " , "Market Analysis:"),
        ("Write an example of a detailed report for a Marketing and Sales Strategy for " , "Marketing and Sales Strategy:"),
        ("Write an example of a detailed report for a Product Development for " , "Product Development:"),
        ("Write an example of a detailed report for a Operations and Management for " , "Operations and Management:"),
        ("Write an example of a detailed report for a Financial Projections for " , "Financial Projections:"),
        ("Explain how this to make this product unique from competitors:", "Considerations:"),
    ],
    "Programming Pattern Prompts": [
        ("Demonstrate a builder pattern in go:", ""),
        ("Demonstrate zero cost abstractions in go:", ""),
        ("Demonstrate a builder pattern in rust:", ""),
        ("Demonstrate Polymorphism in rust:", ""),
        ("Explain how RAII pattern affects rust:", ""),
        ("Demonstrate a builder pattern in c++:", ""),
        ("Explain when to consider using a builder pattern in go:", ""),
        ("Demonstrate a factory pattern in go:", ""),
        ("Explain the use of trait objects in rust:", ""),
        ("Demonstrate a singleton pattern in rust:", ""),
        ("Explain how to implement the strategy pattern in c++:", ""),
        ("Demonstrate a decorator pattern in go:", ""),
        ("Explain the observer pattern in rust:", ""),
        ("Demonstrate a command pattern in c++:", ""),
        ("Explain when to consider using a factory pattern in go:", ""),
        ("Demonstrate a prototype pattern in rust:", ""),
        ("Explain the use of lifetimes in rust and their impact on patterns:", ""),
        ("Demonstrate an adapter pattern in c++:", ""),
        ("Explain the difference between a decorator and a proxy pattern in go:", ""),
        ("Demonstrate a flyweight pattern in rust:", ""),
        ("Explain how to implement the iterator pattern in c++:", ""),
        ("Demonstrate a chain of responsibility pattern in go:", ""),
        ("Explain the use of smart pointers in c++ and their impact on patterns:", ""),
        ("Demonstrate a mediator pattern in rust:", ""),
        ("Explain when to consider using a singleton pattern in go:", ""),
        ("Demonstrate a memento pattern in c++:", ""),
        ("Explain the state pattern in rust:", ""),
        ("Demonstrate a visitor pattern in go:", ""),
        ("Explain how to implement the template method pattern in c++:", ""),
        ("Explain how the observer pattern works in c++:", ""),
        ("Explain the advantages of using a facade pattern in rust:", ""),
        ("Explain how the template method pattern can be used in go:", ""),
        ("Demonstrate a strategy pattern in rust:", ""),
        ("Explain the benefits of using a command pattern in c++:", ""),
        ("Demonstrate a proxy pattern in go:", ""),
        ("Explain how the chain of responsibility pattern works in rust:", ""),
        ("Demonstrate a bridge pattern in c++:", ""),
        ("Explain when to use a mediator pattern in go:", ""),
        ("Explain the advantages of using a composite pattern in c++:", ""),
        ("Explain how the state pattern can be used in rust:", ""),
        ("Explain the benefits of using an iterator pattern in go:", ""),
        ("Demonstrate a memento pattern in rust:", ""),
        ("Explain how the interpreter pattern works in c++:", ""),
        ("Demonstrate a null object pattern in go:", ""),
        ("Explain when to consider using a dependency injection pattern in rust:", ""),
        ("Demonstrate a fluent interface pattern in c++:", ""),
        ("Explain the advantages of using a repository pattern in go:", ""),
        ("Demonstrate a circuit breaker pattern in rust:", ""),
        ("Explain how the throttling pattern can be used in c++:", ""),
        ("Demonstrate a retry pattern in go:", ""),
        ("Explain the benefits of using a bulkhead pattern in rust:", ""),
        ("Demonstrate a CQRS pattern in c++:", ""),
        ("Explain when to use an event sourcing pattern in go:", ""),
        ("Demonstrate a saga pattern in rust:", ""),
        ("Explain how the two-phase commit pattern works in c++:", ""),
    ],
    "Creativity Prompts (Rule observation)": [
        ("Mention things not stated in the following text:", "Unconsidered, Unmentioned"),
        ("Make the following text more vague:", "Vague version:"),
        ("Turn the following text into a bunch of rules:", "Rules:"),
        ("What Syllogisms can be made from this text:", "Syllogisms:"),
        ("Reimagine the following text:", ""),
        ("Extrapolate future scenarios based on the following text:", "Future scenarios:"),
        ("Compare and contrast the perspectives in this text with those from another text:", "Comparison:"),
        ("Design a debate based on the arguments presented in this text:", "Debate design:"),
        ("Create a flowchart that outlines the decision-making process described in this text:", "Flowchart representation:"),
        ("Transform the main ideas of this text into a board game concept:", "Board game concept:"),
        ("Identify the underlying assumptions in the following text:", "Assumptions:"),
        ("Rewrite the following text from a different perspective:", "Alternate perspective:"),
        ("Summarize the main points of the following text in a haiku:", "Haiku summary:"),
        ("Create a metaphor for the main idea of the following text:", "Metaphor:"),
        ("Identify the logical fallacies in the following text:", "Logical fallacies:"),
        ("Rewrite the following text as a dialogue between two characters:", "Dialogue:"),
        ("Create a visual representation of the following text:", "Visual representation:"),
        ("Identify the emotions conveyed in the following text:", "Emotions:"),
        ("Rewrite the following text in the style of a fairy tale:", "Fairy tale version:"),
        ("Create a series of questions that challenge the ideas in the following text:", "Challenging questions:"),
        ("Identify the cultural context and biases in the following text:", "Cultural context and biases:"),
        ("Rewrite the following text as a news article:", "News article:"),
        ("Create a poem inspired by the following text:", "Inspired poem:"),
        ("Identify the implications and consequences of the ideas in the following text:", "Implications and consequences:"),
        ("Rewrite the following text as a series of tweets:", "Tweet series:"),
        ("Create a short story that expands on the following text:", "Short story:"),
        ("Identify the target audience for the following text:", "Target audience:"),
        ("Rewrite the following text as a persuasive speech:", "Persuasive speech:"),
        ("Create a series of emojis that represent the main ideas of the following text:", "Emoji representation:"),
        ("Identify the historical context of the following text:", "Historical context:"),
        ("Rewrite the following text as a scientific abstract:", "Scientific abstract:"),
        ("Create a series of memes inspired by the following text:", "Meme series:"),
        ("Identify the ethical considerations related to the following text:", "Ethical considerations:"),
        ("Create a parody of the following text:", "Parody:"),
        ("Identify the subtext and hidden meanings in the following text:", "Subtext and hidden meanings:"),
        ("Find the implicit assumptions in the following text:", "Implicit assumptions:"),
        ("Make the following text more abstract:", "Abstract version:"),
        ("What are the potential consequences of the following text?:", "Consequences:"),
        ("Rewrite the following text using metaphors:", "Metaphor version:"),
        ("What questions does the following text raise?:", "Questions raised:"),
        ("Identify any cause-and-effect relationships in the following text:", "Cause-and-effect:"),
        ("Make the following text more precise:", "Precise version:"),
        ("What are the underlying values expressed in the following text?:", "Underlying values:"),
        ("Rewrite the following text using analogies:", "Analogy version:"),
        ("What are the potential implications of the following text?:", "Implications:"),
        ("Identify any paradoxes in the following text:", "Paradoxes:"),
        ("Rewrite the following text using personification:", "Personification version:"),
        ("What are the potential biases in the following text?:", "Biases:"),
    ],
    "Game Based" : [
        {"Write a story in 10 short sentences (6 words or less):", "1. "},
        {"Suggest ways that the following one sentence scenario can be delayed or complicated:", "1. "},
        {"Suggest ways that complication in a story can be avoided/subverted:", "Strategies: "},
        ("What obstacles to growth for a protagonist exist in the following text:", "Obstacles:"),
        ("Write a story for the basis of a random game", "Story:"),
        ("What are common themes in games?", ""),
        ("Write Three factions and why they are at conflict based on the following text:", "Faction 1:"),
    ],
    "Random Logic and Machine Learning 100624": [
        {"What concepts are similar to counter intuition?", ""},
        {"Please explain PvsNP problem?", ""},
        {"With the context of this text, what would be the PvsNP equivalent", ""},
        {"What does out of distribution mean and what algorithms can make it?", ""},
        {"List some existing evolutionary algorithms and explain how they relate to the term out-of distribution", ""},
        {"Please help me understand ternary vs binary calculations", ""},
        {"What is an auto encoder and what are hyperparameters", ""},
        {"Please help me make a glossary from this text", ""},
        {"Please list as many indefinite words as you can:", ""},
        {"What is the difference between indefinite words and hypernyms?", ""},
        {"What concepts are similar to counter intuition?", ""},
        {"'When stuck, looking for the wrong way to do something is easier and more productive than looking for the right way' - please critique this statement", ""},
        {"What makes long division hard when there are decimals?", "\n"},
        {"What tasks are non-linear in nature?", "\n"},
    ]
}

AllWorkflows = {
    "Merged Spelling": [
        "Please make an image description the meaning of the word much",
        "next please find english words close in spelling to henduo",
        "great, next make an image description using those words",
        "Thanks lets now merge the two seperate image descriptions together", 
    ]
}

with gr.Blocks() as iface:
    with gr.Tab("Single prompt"):
        gr.HTML("<a href='https://huggingface.co/spaces/stabilityai/stablelm-2-1_6b-zephyr'> -- Original StabilityAI demo -- </a> | To be tested later - <a href='https://huggingface.co/spaces/Qwen/Qwen2-0.5B-Instruct'> -- Qwen2 0.5B Instruct -- </a> | DeepSeek-Coder-V2-Lite-Instruct-Q3_K_M.gguf  += 8gb so test for how big the context can be and how speed is affected  ")
        gr.Interface(
            fn=generate_response,
            inputs=gr.Textbox(lines=2, placeholder="Type your message here..."),
            outputs="text",
            title="Stable LM 2 Zephyr (1.6b) LLama.cpp Interface Test (Inconsistent Performance - 100 tokens in 50 secs (when this HF space is updated can even go 6tokens a second for non formatted text) or 800+ secs(HF space open for long))",
            description="No Prompt template used yet (Essentially autocomplete). No Message History for now - Enter your message and get a response.",
            flagging_dir="/usr/src/app/flagged",
        )

        gr.HTML("Any standard way of thinking / Repetitive idea / rule of thumb / advice can be turned into a button (In a timeline?)")
        gr.HTML("LLM powered Buttons as the new notetaking? (Youtube Video to prompt pipeline?)<br><br>List to buttons (Instead of buttons tabs and dropdowns maybe?)")

        MainOutput = gr.TextArea(placeholder='Output will show here')
        CustomButtonInput = gr.TextArea(lines=1, placeholder='Prompt goes here')

        for category_name, category_prompts in Allprompts.items():
            with gr.Accordion(f"General {category_name} Pattern based", open=False):
                with gr.Group():
                    for index, (prompt, _) in enumerate(category_prompts):
                        button = gr.Button(prompt)
                        button.click(custom_generate_response, inputs=[CustomButtonInput, gr.State(index), gr.State(category_name)], outputs=MainOutput)


    with gr.Tab("Workflow Brainstom"):
        gr.HTML("Workflow = premeditated events --- need a timeline before prompts --- eg. old conversation that needs to be repeated over and over again")


iface.queue().launch(server_name="0.0.0.0", share=True)