Upload 4 files

README.md

@@ -1,13 +1,35 @@
----
-title: Token Counter
-emoji: 🌖
-colorFrom: blue
-colorTo: purple
-sdk: streamlit
-sdk_version: 1.19.0
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Token Counter
+
+![Release status](https://img.shields.io/badge/status-beta-blue?label=Release%20Status&style=plastic)
+![GitHub last commit](https://img.shields.io/github/last-commit/LightningRalf/token_counter)
+![GitHub repo size](https://img.shields.io/github/repo-size/LightningRalf/token_counter)
+![GitHub issues](https://img.shields.io/github/issues/LightningRalf/token_counter)
+![GitHub](https://img.shields.io/github/license/LightningRalf/token_counter)
+
+Token Counter is a simple Python script that counts the number of tokens in a Markdown file. It's useful for analyzing and processing text data in natural language processing tasks.
+
+## Installation
+
+To use Token Counter, simply clone the repository:
+
+```bash
+git clone https://github.com/LightningRalf/token_counter.git
+```
+
+## Usage
+
+To count the tokens in a Markdown file, run the `token_counter.py` script with the file path as an argument:
+
+```bash
+python token_counter.py path/to/your/markdown_file.md
+```
+
+The script will print the token count and also log the results in a log file.
+
+## Contributing
+
+We welcome contributions to improve Token Counter! Please feel free to open an issue or submit a pull request if you have any suggestions or improvements.
+
+## License
+
+This project is licensed under the CC0-1.0 License - see the [LICENSE](LICENSE) file for details.

app.py

@@ -0,0 +1,35 @@
+import streamlit as st
+from transformers import AutoTokenizer
+import requests
+import datetime
+from dateutil.relativedelta import relativedelta
+
+# Count tokens in a text string using a specified language model.
+def count_tokens_text(text, model_name='gpt4'):
+    # (same as before)
+
+# Fetch the most popular models from the last month
+def get_popular_models():
+    one_month_ago = (datetime.datetime.now() - relativedelta(months=1)).strftime("%Y-%m-%d")
+    api_url = f"https://huggingface.co/api/models?sort=downloads&direction=desc&start_date={one_month_ago}"
+    response = requests.get(api_url)
+    data = response.json()
+    popular_models = [model["modelId"] for model in data["results"]]
+    return popular_models
+
+# Streamlit app
+st.title("Token Counter")
+text = st.text_area("Text:", value="", height=200)
+
+popular_models = get_popular_models()
+model_name = st.selectbox("Model:", options=popular_models, index=0)
+manual_entry = st.text_input("Or enter a model manually:", value="")
+if manual_entry:
+    model_name = manual_entry
+
+if st.button("Count Tokens"):
+    token_count, error = count_tokens_text(text, model_name)
+    if token_count is not None:
+        st.success(f"Token count: {token_count}")
+    elif error is not None:
+        st.error(f"Error: {error}")

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+transformers
+requests
+datetime
+dateutil.relativedelta

token_counter.log

@@ -0,0 +1 @@
+2023-04-29 10:16:15,841 - INFO - Token count for C:\Users\mjpa\Documents\Obsidian\20-29_Projekte\21_jPAw\21.96_MultiAgentSystem\OBJECTIVE-MAS-GITHUB-basedOnAgentLLM.md: 227