initial commit

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+__pycache__
+
+.env

README.md

@@ -0,0 +1,26 @@
+---
+title: Write4All
+emoji: ✍️✅
+colorFrom: pink
+colorTo: green
+sdk: gradio
+sdk_version: 4.25.0
+app_file: app.py
+pinned: true
+license: mit 
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+## Running the app
+
+```bash
+poetry install
+poetry run gradio app.py
+```
+
+Update the requirements.txt file with the following command:
+```bash
+poetry export --without-hashes --without dev -f requirements.txt -o requirements.txt
+```

app.py

@@ -0,0 +1,191 @@
+import re
+import os
+import gradio as gr
+import json
+from functools import cache
+
+import google.generativeai as genai
+
+
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv()
+except:
+    pass
+
+generation_config = {
+    "temperature": 0.9,  # Temperature of the sampling distribution
+    "top_p": 1,  # Probability of sampling from the top p tokens
+    "top_k": 1,  # Number of top tokens to sample from
+    "max_output_tokens": 2048,
+}
+
+safety_settings = [
+    {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},
+    {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},
+    {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
+    {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_ONLY_HIGH"},
+]
+
+genai.configure(api_key=os.getenv("GEMINI_API_KEY"))
+
+text_model = genai.GenerativeModel(
+    model_name="gemini-1.0-pro",
+    generation_config=generation_config,
+    safety_settings=safety_settings,
+)
+vision_model = genai.GenerativeModel(
+    "gemini-pro-vision",
+    generation_config=generation_config,
+    safety_settings=safety_settings,
+)
+
+
+@cache
+def get_file(path: str) -> str:
+    with open(path) as f:
+        return f.read()
+
+
+def fix_json(json_str: str) -> str:
+    template = get_file("templates/prompt_json_fix.txt")
+    prompt = template.format(json=json_str)
+    response = text_model.generate_content(prompt).text
+    return response.split("```json")[1].split("```")[0]
+
+
+def get_json_content(response: str) -> dict:
+    print(response)
+    if "```json" not in response:
+        return []
+    raw_json = response.split("```json")[1].split("```")[0]
+    try:
+        return json.loads(raw_json)
+    except json.JSONDecodeError as e:
+        print(e)
+        new_json = fix_json(raw_json)
+        print(new_json)
+        return json.loads(new_json)
+
+
+def review_text(text: str) -> list[dict]:
+    template = get_file("templates/prompt_v1.txt")
+    try:
+        response = text_model.generate_content(template.format(text=text)).text
+    except ValueError as e:
+        print(e)
+        raise ValueError(
+            f"Error while getting answer from the model, make sure the content isn't offensive or dangerous."
+        )
+    return get_json_content(response)
+
+
+def review_image(image) -> list[dict]:
+    prompt = get_file("templates/prompt_image_v1.txt")
+    try:
+        response = vision_model.generate_content([prompt, image]).text
+    except ValueError as e:
+        print(e)
+        message = "Error while getting answer from the model, make sure the content isn't offensive or dangerous. Please try again or change the prompt."
+        gr.Error(message)
+        raise ValueError(message)
+    return response
+
+
+def html_title(title: str) -> str:
+    return f"<h1>{title}</h1>"
+
+
+def apply_review(text: str, review: list[dict]) -> str:
+    output = ""
+    review = sorted(review, key=lambda x: x["start_char"])
+    last_end = 0
+    for entity in review:
+        starts = [
+            m.start() + last_end
+            for m in re.finditer(entity["term"].lower(), text[last_end:].lower())
+        ]
+        if len(starts) > 0:
+            start = starts[0]
+            end = start + len(entity["term"])
+            output += text[last_end:start]
+            output += get_file("templates/correction.html").format(
+                term=text[start:end], fix=entity["fix"], kind=entity["type"]
+            )
+            last_end = end
+    output += text[last_end:]
+    return output
+
+
+def review_table_summary(review: list[dict]) -> str:
+    table = "<table><tr><th>Term</th><th>Fix</th><th>Type</th><th>Reason</th></tr>"
+    for entity in review:
+        table += f"<tr><td>{entity['term']}</td><td>{entity['fix']}</td><td>{entity['type']}</td><td>{entity.get('reason', '-')}</td></tr>"
+    table += "</table>"
+    return table
+
+
+def format_entities(text: str, review: list[dict]) -> list[dict]:
+    entities = []
+    for entity in review:
+        # Find all occurrences of the term in the text
+        starts = [m.start() for m in re.finditer(entity["term"], text)]
+        if len(starts) > 0:
+            entities.append(
+                {
+                    "term": entity["term"],
+                    "start": starts[0],
+                    "end": starts[0] + len(entity["term"]),
+                    "entity": entity["type"],
+                    "fix": entity["fix"],
+                }
+            )
+        else:
+            print(f"Term '{entity['term']}' not found in the text: '{text}'")
+    return entities
+
+
+def process_text(text):
+    review = review_text(text)
+    if len(review) == 0:
+        return html_title("No issues found in the text 🎉🎉🎉")
+    return (
+        html_title("Reviewed text")
+        + apply_review(text, review)
+        + html_title("Explanation")
+        + review_table_summary(review)
+    )
+
+
+def process_image(image):
+    print(image)
+    return review_image(image)
+
+
+text_ui = gr.Interface(
+    fn=process_text,
+    inputs=["text"],
+    outputs=[gr.HTML(label="Revision")],
+    examples=[
+        "The whitelist is incomplete.",
+        "There's not enough manpower to deliver the project",
+        "This has never happened in the history of mankind!",
+        "El hombre desciende del mono.",
+        "Els homes són animals",
+    ],
+)
+
+image_ui = gr.Interface(
+    fn=process_image,
+    inputs=gr.Image(sources=["upload", "clipboard"], type="pil"),
+    outputs=["markdown"],
+    examples=["static/images/CEOs.png", "static/images/meat_grid.png"],
+)
+
+with gr.Blocks() as demo:
+    gr.Markdown(get_file("static/intro.md"))
+    gr.TabbedInterface([text_ui, image_ui], ["Check texts", "Check images"])
+
+if __name__ == "__main__":
+    demo.launch()

pyproject.toml

@@ -0,0 +1,19 @@
+[tool.poetry]
+name = "inclu-v1"
+version = "0.1.0"
+description = ""
+authors = ["Jaume Ferrarons <jaume.ferrarons@gmail.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+gradio = "^4.25.0"
+google-generativeai = "^0.4.1"
+
+
+[tool.poetry.group.dev.dependencies]
+python-dotenv = "^1.0.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,82 @@
+aiofiles==23.2.1 ; python_version >= "3.10" and python_version < "4.0"
+altair==5.3.0 ; python_version >= "3.10" and python_version < "4.0"
+annotated-types==0.6.0 ; python_version >= "3.10" and python_version < "4.0"
+anyio==4.3.0 ; python_version >= "3.10" and python_version < "4.0"
+attrs==23.2.0 ; python_version >= "3.10" and python_version < "4.0"
+cachetools==5.3.3 ; python_version >= "3.10" and python_version < "4.0"
+certifi==2024.2.2 ; python_version >= "3.10" and python_version < "4.0"
+charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "4.0"
+click==8.1.7 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and platform_system == "Windows"
+contourpy==1.2.1 ; python_version >= "3.10" and python_version < "4.0"
+cycler==0.12.1 ; python_version >= "3.10" and python_version < "4.0"
+exceptiongroup==1.2.0 ; python_version >= "3.10" and python_version < "3.11"
+fastapi==0.110.1 ; python_version >= "3.10" and python_version < "4.0"
+ffmpy==0.3.2 ; python_version >= "3.10" and python_version < "4.0"
+filelock==3.13.3 ; python_version >= "3.10" and python_version < "4.0"
+fonttools==4.50.0 ; python_version >= "3.10" and python_version < "4.0"
+fsspec==2024.3.1 ; python_version >= "3.10" and python_version < "4.0"
+google-ai-generativelanguage==0.4.0 ; python_version >= "3.10" and python_version < "4.0"
+google-api-core==2.18.0 ; python_version >= "3.10" and python_version < "4.0"
+google-api-core[grpc]==2.18.0 ; python_version >= "3.10" and python_version < "4.0"
+google-auth==2.29.0 ; python_version >= "3.10" and python_version < "4.0"
+google-generativeai==0.4.1 ; python_version >= "3.10" and python_version < "4.0"
+googleapis-common-protos==1.63.0 ; python_version >= "3.10" and python_version < "4.0"
+gradio-client==0.15.0 ; python_version >= "3.10" and python_version < "4.0"
+gradio==4.25.0 ; python_version >= "3.10" and python_version < "4.0"
+grpcio-status==1.62.1 ; python_version < "4.0" and python_version >= "3.10"
+grpcio==1.62.1 ; python_version < "4.0" and python_version >= "3.10"
+h11==0.14.0 ; python_version >= "3.10" and python_version < "4.0"
+httpcore==1.0.5 ; python_version >= "3.10" and python_version < "4.0"
+httpx==0.27.0 ; python_version >= "3.10" and python_version < "4.0"
+huggingface-hub==0.22.2 ; python_version >= "3.10" and python_version < "4.0"
+idna==3.6 ; python_version >= "3.10" and python_version < "4.0"
+importlib-resources==6.4.0 ; python_version >= "3.10" and python_version < "4.0"
+jinja2==3.1.3 ; python_version >= "3.10" and python_version < "4.0"
+jsonschema-specifications==2023.12.1 ; python_version >= "3.10" and python_version < "4.0"
+jsonschema==4.21.1 ; python_version >= "3.10" and python_version < "4.0"
+kiwisolver==1.4.5 ; python_version >= "3.10" and python_version < "4.0"
+markdown-it-py==3.0.0 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "4.0"
+matplotlib==3.8.3 ; python_version >= "3.10" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+numpy==1.26.4 ; python_version >= "3.10" and python_version < "4.0"
+orjson==3.10.0 ; python_version >= "3.10" and python_version < "4.0"
+packaging==24.0 ; python_version >= "3.10" and python_version < "4.0"
+pandas==2.2.1 ; python_version >= "3.10" and python_version < "4.0"
+pillow==10.3.0 ; python_version >= "3.10" and python_version < "4.0"
+proto-plus==1.23.0 ; python_version >= "3.10" and python_version < "4.0"
+protobuf==4.25.3 ; python_version >= "3.10" and python_version < "4.0"
+pyasn1-modules==0.4.0 ; python_version >= "3.10" and python_version < "4.0"
+pyasn1==0.6.0 ; python_version >= "3.10" and python_version < "4.0"
+pydantic-core==2.16.3 ; python_version >= "3.10" and python_version < "4.0"
+pydantic==2.6.4 ; python_version >= "3.10" and python_version < "4.0"
+pydub==0.25.1 ; python_version >= "3.10" and python_version < "4.0"
+pygments==2.17.2 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+pyparsing==3.1.2 ; python_version >= "3.10" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "4.0"
+python-multipart==0.0.9 ; python_version >= "3.10" and python_version < "4.0"
+pytz==2024.1 ; python_version >= "3.10" and python_version < "4.0"
+pyyaml==6.0.1 ; python_version >= "3.10" and python_version < "4.0"
+referencing==0.34.0 ; python_version >= "3.10" and python_version < "4.0"
+requests==2.31.0 ; python_version >= "3.10" and python_version < "4.0"
+rich==13.7.1 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+rpds-py==0.18.0 ; python_version >= "3.10" and python_version < "4.0"
+rsa==4.9 ; python_version >= "3.10" and python_version < "4"
+ruff==0.3.5 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+semantic-version==2.10.0 ; python_version >= "3.10" and python_version < "4.0"
+shellingham==1.5.4 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+six==1.16.0 ; python_version >= "3.10" and python_version < "4.0"
+sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0"
+starlette==0.37.2 ; python_version >= "3.10" and python_version < "4.0"
+tomlkit==0.12.0 ; python_version >= "3.10" and python_version < "4.0"
+toolz==0.12.1 ; python_version >= "3.10" and python_version < "4.0"
+tqdm==4.66.2 ; python_version >= "3.10" and python_version < "4.0"
+typer-cli==0.12.0 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+typer-slim[standard]==0.12.0 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+typer[all]==0.12.0 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+typing-extensions==4.10.0 ; python_version >= "3.10" and python_version < "4.0"
+tzdata==2024.1 ; python_version >= "3.10" and python_version < "4.0"
+urllib3==2.2.1 ; python_version >= "3.10" and python_version < "4.0"
+uvicorn==0.29.0 ; python_version >= "3.10" and python_version < "4.0" and sys_platform != "emscripten"
+websockets==11.0.3 ; python_version >= "3.10" and python_version < "4.0"

static/intro.md

@@ -0,0 +1,5 @@
+# ✅ Write4All - Create the right for everyone
+
+Use ✅ Write4All to review your content and make sure it is respectful and inclusive. Use the results under taking into account the context and decide if you finally need to make proposed changes.
+
+> Terms of use: You won't use this app to produce harmful content, promote hate speech, violence, or discrimination. You agree to the terms of use of Gemini API. By using this app, you agree to the terms of use of the models used in this app.

templates/correction.html

@@ -0,0 +1 @@
+<span title="{kind}"><span style='background-color:#FF5555; color:black; text-decoration:line-through; text-decoration-thickness: 2px; min-height: 22px; display: inline-block; border-radius: 5px 0 0 5px; padding-left: 5px;'>{term}</span><span style='background-color:#55FF55; color:black; min-height: 22px; display: inline-block; border-radius: 0 5px 5px 0; padding-right: 5px;'>{fix}</span></span>

templates/prompt_image_v1 copy.txt

@@ -0,0 +1,8 @@
+Analyze the following image in terms of inclusivity.
+
+You must:
+- Provide a brief description of the image and explain why it is inclusive or exclusive.
+- When the picture is exclusive make sure to provide a table with to columns: "problem" and "solution". In the problem column, list the issues with the image. In the solution column, provide a solution to the problem.
+- When the image is inclusive, provide a brief explanation of why it is inclusive.
+- In case the picture contains people, consider inclusivity in the following dimensions: gender, ethnicity, age, ability, etc.
+- Only talk about people when there is people in the picture.

templates/prompt_image_v1.txt

@@ -0,0 +1,4 @@
+Analyze the following image in terms of inclusivity.
+
+Provide a brief description of the image and explain how it is inclusive or exclusive. Then if it's exclusive provide a table with to columns: "problem" and "solution". In the problem column, list the issues with the image. In the solution column, provide a solution to the problem.
+Otherwise, if the image is inclusive, provide a brief explanation of why it is inclusive.

templates/prompt_json_fix.txt

@@ -0,0 +1,3 @@
+Could you fix the following JSON?
+
+{json}

templates/prompt_v1.txt

@@ -0,0 +1,58 @@
+Provide tips to improve the following text regarding inclusivity. Use the following classification:
+ - "gender-exclusive": use of terms that exclude non-binary or gender-diverse individuals
+ - "stereotyping": Making assumptions based on characteristics like race, gender, or disability.
+ - "ableism": Stigmatizing disabilities or mental health conditions.
+ - "religion/culture": Using language that offends specific religious or cultural groups.
+ - "body-stigma": Perpetuating negative stereotypes about body size and appearance.
+ - "microagressions": Unintentional comments that marginalize or offend others.
+
+You must:
+ - provide a single fix for each issue.
+ - the term should be as short as possible while including all the text that has to be replaced.
+ - ensure that replacing the term by the fix does not change the meaning of the text and lead to a grammatically correct sentence.
+ - briefly reason why the term is problematic.
+
+Input:
+```
+The policeman stopped the car
+```
+
+Output:
+```json
+[
+  {{
+    "term": "policeman",
+    "start_char": 4,
+    "end_char": 13,
+    "type": "gender-exclusive",
+    "fix": "police officer",
+    "reason": "police officer is gender neutral and not assuming the gender of the subject"
+  }}
+]
+```
+
+Input:
+```
+People who are overweight eat too much.
+```
+
+Output:
+```json
+[
+  {{
+    "term": "People who are overweight eat too much.",
+    "start_char": 0,
+    "end_char": 39,
+    "type": "body-stigma",
+    "fix": "Some people that eat too much are overweight.",
+    "reason": "Overweight can be a medical condition and not always related to eating to much"
+  }}
+]
+```
+
+Input:
+```
+{text}
+```
+
+Output: