Upload 43 files

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [claromes]

.github/workflows/docs.yml

@@ -0,0 +1,34 @@
+name: docs
+
+on: [push, pull_request, workflow_dispatch]
+
+permissions:
+  contents: write
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+      - name: Install dependencies
+        run: |
+          poetry install
+      - name: Sphinx build
+        run: |
+          mkdir gh-pages
+          touch gh-pages/.nojekyll
+          cd docs/
+          poetry run sphinx-build -b html . _build
+          cp -r _build/* ../gh-pages/
+      - name: Deploy documentation
+        if: ${{ github.event_name == 'push' }}
+        uses: JamesIves/github-pages-deploy-action@4.1.4
+        with:
+          branch: gh-pages
+          folder: gh-pages

.streamlit/config.toml

@@ -0,0 +1,13 @@
+[theme]
+base = "light"
+primaryColor = "black"
+secondaryBackgroundColor = "gainsboro"
+textColor = "black"
+backgroundColor = "whitesmoke"
+font = "serif"
+
+[client]
+toolbarMode = "minimal"
+
+[server]
+port = 8501

app/app.py

@@ -0,0 +1,374 @@
+import base64
+from datetime import datetime, timedelta
+
+import streamlit as st
+import streamlit.components.v1 as components
+
+from waybacktweets.api.export import TweetsExporter
+from waybacktweets.api.parse import TweetsParser
+from waybacktweets.api.request import WaybackTweets
+from waybacktweets.api.visualize import HTMLTweetsVisualizer
+from waybacktweets.config import FIELD_OPTIONS, config
+
+# ------ Initial Settings ------ #
+
+PAGE_ICON = "assets/parthenon.png"
+TITLE = "assets/waybacktweets.png"
+DOWNLOAD = "assets/download.svg"
+
+collapse = None
+matchtype = None
+start_date = datetime.now() - timedelta(days=30 * 6)
+end_date = datetime.now()
+min_date = datetime(2006, 1, 1)
+
+# ------ Verbose Mode Configuration ------ #
+
+config.verbose = False
+
+# ------ Page Configuration ------ #
+
+st.set_page_config(
+    page_title="Wayback Tweets",
+    page_icon=PAGE_ICON,
+    layout="centered",
+    menu_items={
+        "About": f"""
+    [![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/claromes/waybacktweets?include_prereleases)](https://github.com/claromes/waybacktweets/releases) [![License](https://img.shields.io/github/license/claromes/waybacktweets)](https://github.com/claromes/waybacktweets/blob/main/LICENSE.md) [![Star](https://img.shields.io/github/stars/claromes/waybacktweets?style=social)](https://github.com/claromes/waybacktweets)
+
+    The application is a prototype hosted on Streamlit Cloud, serving as an alternative to the command line tool.
+
+    © 2023 - {end_date.year}, [Claromes](https://claromes.com)
+
+    ---
+""",  # noqa: E501
+        "Report a bug": "https://github.com/claromes/waybacktweets/issues",
+    },
+)
+
+# ------ Set States and Params ------ #
+
+if "current_username" not in st.session_state:
+    st.session_state.current_username = ""
+
+if "count" not in st.session_state:
+    st.session_state.count = False
+
+if "archived_timestamp_filter" not in st.session_state:
+    st.session_state.archived_timestamp_filter = (start_date, end_date)
+
+if "username_value" not in st.session_state:
+    st.session_state.username_value = ""
+
+if "expanded_value" not in st.session_state:
+    st.session_state.expanded_value = False
+
+if "query" not in st.session_state:
+    st.session_state.query = False
+
+if "update_component" not in st.session_state:
+    st.session_state.update_component = 0
+
+if "username" not in st.query_params:
+    st.query_params["username"] = ""
+
+# ------ Add Custom CSS Style ------ #
+
+st.html(
+    """
+    <style>
+        header[data-testid="stHeader"] {
+            opacity: 0.5;
+        }
+        iframe {
+            border: 1px solid #dddddd;
+            border-radius: 0.5rem;
+        }
+        div[data-testid="InputInstructions"] {
+            visibility: hidden;
+        }
+        button[data-testid="StyledFullScreenButton"] {
+            display: none;
+        }
+        div[class="st-emotion-cache-1v0mbdj e115fcil1"] {
+            max-width: 100%;
+        }
+    </style>
+    """
+)
+
+# ------ Requestings ------ #
+
+
+@st.cache_data(ttl=600, show_spinner=False)
+def wayback_tweets(
+    username,
+    collapse,
+    timestamp_from,
+    timestamp_to,
+    limit,
+    offset,
+    matchtype,
+):
+    response = WaybackTweets(
+        username,
+        collapse,
+        timestamp_from,
+        timestamp_to,
+        limit,
+        offset,
+        matchtype,
+    )
+    archived_tweets = response.get()
+
+    return archived_tweets
+
+
+@st.cache_data(ttl=600, show_spinner=False)
+def tweets_parser(archived_tweets, username, field_options):
+    parser = TweetsParser(archived_tweets, username, field_options)
+    parsed_tweets = parser.parse()
+
+    return parsed_tweets
+
+
+@st.cache_data(ttl=600, show_spinner=False)
+def tweets_exporter(parsed_tweets, username, field_options):
+    exporter = TweetsExporter(parsed_tweets, username, field_options)
+
+    df = exporter.dataframe
+    file_name = exporter.filename
+
+    return df, file_name
+
+
+# ------ Custom JavaScript ------ #
+
+
+def scroll_page():
+    js = f"""
+    <script>
+        window.parent.document.querySelector('section.main').scrollTo(700, 700);
+        let update_component = {st.session_state.update_component} // Force component update to generate scroll
+    </script>
+    """  # noqa: E501
+
+    components.html(js, width=0, height=0)
+
+
+# ------ Query Param ------ #
+
+if st.query_params.username != "":
+    st.session_state.username_value = st.query_params.username
+    st.session_state.expanded_value = True
+    st.session_state.query = True
+
+    st.session_state.update_component += 1
+    scroll_page()
+
+# ------ User Interface Settings ------ #
+
+st.info(
+    "🥳 [**Pre-release 1.0x: Python module, CLI, and new Streamlit app**](https://github.com/claromes/waybacktweets/releases)"  # noqa: E501
+)
+
+st.image(TITLE, use_column_width="never")
+st.caption(
+    "[![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/claromes/waybacktweets?include_prereleases)](https://github.com/claromes/waybacktweets/releases) [![Star](https://img.shields.io/github/stars/claromes/waybacktweets?style=social)](https://github.com/claromes/waybacktweets)"  # noqa: E501
+)
+st.write(
+    "Retrieves archived tweets CDX data in HTML (for easy viewing of the tweets using the `iframe` tag), CSV, and JSON formats."  # noqa: E501
+)
+
+st.write(
+    "This application uses the Wayback Tweets Python package, which can be used either as a module or as a standalone command-line tool. [Read the documentation](https://claromes.github.io/waybacktweets) for more information."  # noqa: E501
+)
+
+st.write(
+    "To access the legacy version of Wayback Tweets, [click here](https://waybacktweets-legacy.streamlit.app)."  # noqa: E501
+)
+
+st.divider()
+
+# -- Filters -- #
+
+username = st.text_input(
+    "Username *",
+    value=st.session_state.username_value,
+    key="username",
+    placeholder="Without @",
+)
+
+with st.expander("Filtering", expanded=st.session_state.expanded_value):
+
+    st.session_state.archived_timestamp_filter = st.date_input(
+        "Tweets saved between",
+        (start_date, end_date),
+        min_date,
+        end_date,
+        format="YYYY/MM/DD",
+        help="Using the `from` and `to` filters. Format: YYYY/MM/DD",
+    )
+    st.caption(
+        ":orange[note: large date range takes a long time to process, and the app's resources may not be sufficient. Try to perform searches with smaller ranges to get faster results.]"  # noqa: E501
+    )
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        limit = st.text_input(
+            "Limit",
+            key="limit",
+            help="Query result limits",
+        )
+
+    with col2:
+        offset = st.text_input(
+            "Offset",
+            key="offset",
+            help="Allows for a simple way to scroll through the results",
+        )
+
+    unique = st.checkbox(
+        "Only unique Wayback Machine URLs",
+        key="unique",
+        help="Filtering by the collapse option using the `urlkey` field and the URL Match Scope `prefix`",  # noqa: E501
+    )
+    st.caption(
+        ":orange[note: according to the official documentation of the Wayback CDX Server API, the query to retrieve unique URLs may be slow at the moment.]"  # noqa: E501
+    )
+
+
+query = st.button("Query", type="primary", use_container_width=True)
+
+if st.query_params.username == "":
+    st.query_params.clear()
+    st.session_state.query = query
+
+# ------ Results ------ #
+
+if username != st.session_state.current_username:
+    st.session_state.current_username = username
+
+if st.session_state.query or st.session_state.count:
+    if unique:
+        collapse = "urlkey"
+        matchtype = "prefix"
+
+    try:
+        with st.spinner(
+            f"Waybacking @{st.session_state.current_username}'s archived tweets"
+        ):
+            wayback_tweets = wayback_tweets(
+                st.session_state.current_username,
+                collapse,
+                st.session_state.archived_timestamp_filter[0],
+                st.session_state.archived_timestamp_filter[1],
+                limit,
+                offset,
+                matchtype,
+            )
+
+        if not wayback_tweets:
+            st.error("No data was saved due to an empty response.")
+            st.stop()
+
+        with st.spinner(
+            f"Parsing @{st.session_state.current_username}'s archived tweets"
+        ):
+            parsed_tweets = tweets_parser(
+                wayback_tweets, st.session_state.current_username, FIELD_OPTIONS
+            )
+
+            df, file_name = tweets_exporter(
+                parsed_tweets, st.session_state.current_username, FIELD_OPTIONS
+            )
+
+        csv_data = df.to_csv(index=False)
+        json_data = df.to_json(orient="records", lines=False)
+        html = HTMLTweetsVisualizer(username, json_data)
+        html_content = html.generate()
+
+        # -- Rendering -- #
+
+        if csv_data and json_data and html_content:
+            st.session_state.count = len(df)
+            st.write(f"**{st.session_state.count} URLs have been captured**")
+
+            # -- HTML -- #
+
+            st.header("HTML", divider="gray", anchor=False)
+            st.write(
+                f"Visualize tweets more efficiently through `iframes`. Download the @{st.session_state.current_username}'s archived tweets in HTML."  # noqa: E501
+            )
+
+            col5, col6 = st.columns([1, 18])
+
+            with col5:
+                st.image(DOWNLOAD, width=22)
+
+            with col6:
+                b64_html = base64.b64encode(html_content.encode()).decode()
+                href_html = f"data:text/html;base64,{b64_html}"
+
+                st.markdown(
+                    f'<a href="{href_html}" download="{file_name}.html" title="Download {file_name}.html">{file_name}.html</a>',  # noqa: E501
+                    unsafe_allow_html=True,
+                )
+
+            # -- CSV -- #
+
+            st.header("CSV", divider="gray", anchor=False)
+            st.write(
+                "Check the data returned in the dataframe below and download the file."
+            )
+
+            col7, col8 = st.columns([1, 18])
+
+            with col7:
+                st.image(DOWNLOAD, width=22)
+
+            with col8:
+                b64_csv = base64.b64encode(csv_data.encode()).decode()
+                href_csv = f"data:file/csv;base64,{b64_csv}"
+
+                st.markdown(
+                    f'<a href="{href_csv}" download="{file_name}.csv" title="Download {file_name}.csv">{file_name}.csv</a>',  # noqa: E501
+                    unsafe_allow_html=True,
+                )
+
+            st.dataframe(df, use_container_width=True)
+
+            # -- JSON -- #
+
+            st.header("JSON", divider="gray", anchor=False)
+            st.write(
+                "Check the data returned in JSON format below and download the file."
+            )
+
+            col9, col10 = st.columns([1, 18])
+
+            with col9:
+                st.image(DOWNLOAD, width=22)
+
+            with col10:
+                b64_json = base64.b64encode(json_data.encode()).decode()
+                href_json = f"data:file/json;base64,{b64_json}"
+
+                st.markdown(
+                    f'<a href="{href_json}" download="{file_name}.json" title="Download {file_name}.json">{file_name}.json</a>',  # noqa: E501
+                    unsafe_allow_html=True,
+                )
+
+            st.json(json_data, expanded=False)
+    except TypeError as e:
+        st.error(
+            f"""
+        {e}. Refresh this page and try again.
+
+        If the problem persists [open an issue](https://github.com/claromes/waybacktweets/issues)."""  # noqa: E501
+        )
+        st.stop()
+    except Exception as e:
+        st.error(str(e))
+        st.stop()

app/requirements.txt

@@ -0,0 +1,2 @@
+streamlit==1.36.0
+waybacktweets==1.0a5

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/css/custom.css

@@ -0,0 +1,14 @@
+body {
+    font-family: Georgia, 'Times New Roman', Times, serif;
+    background-color: whitesmoke;
+}
+
+a:hover {
+    background-color: whitesmoke !important;
+}
+
+#cli #usage #waybacktweets h3,
+#cli .admonition-title,
+.sphinxsidebarwrapper li ul li ul:has(a[href="#waybacktweets"]):last-child {
+    display: none;
+}

docs/_templates/page.html

@@ -0,0 +1,14 @@
+{% extends "!page.html" %}
+
+{% block extrahead %}
+{{ super() }}
+    <meta name="description" content="Retrieves archived tweets CDX data from the Wayback Machine, performs necessary parsing, and saves the data">
+
+    <meta property="og:title" content="{{ title|e }}" />
+    <meta property="og:description" content="Retrieves archived tweets CDX data from the Wayback Machine, performs necessary parsing, and saves the data">
+    <meta property="og:image" content="https://claromes.github.io/waybacktweets/_static/card.png" />
+
+    <meta name="twitter:title" content="{{ title|e }}">
+    <meta name="twitter:description" content="Retrieves archived tweets CDX data from the Wayback Machine, performs necessary parsing, and saves the data">
+    <meta property="twitter:image" content="https://claromes.github.io/waybacktweets/_static/card.png" />
+{% endblock %}

docs/api.rst

@@ -0,0 +1,72 @@
+API
+====
+
+Request
+---------
+
+.. automodule:: waybacktweets.api.request
+
+.. autoclass:: WaybackTweets
+    :members:
+
+.. _parser:
+
+Parse
+---------
+
+.. automodule:: waybacktweets.api.parse
+
+.. autoclass:: TweetsParser
+    :members:
+    :private-members:
+
+.. autoclass:: TwitterEmbed
+    :members:
+
+.. autoclass:: JsonParser
+    :members:
+
+.. _exporter:
+
+Export
+---------
+
+.. automodule:: waybacktweets.api.export
+
+.. autoclass:: TweetsExporter
+    :members:
+    :private-members:
+
+Visualize
+-----------
+
+.. automodule:: waybacktweets.api.visualize
+
+.. autoclass:: HTMLTweetsVisualizer
+    :members:
+    :private-members:
+
+.. _utils:
+
+Utils
+-------
+
+.. automodule:: waybacktweets.utils.utils
+
+.. autofunction:: check_double_status
+.. autofunction:: check_pattern_tweet
+.. autofunction:: check_url_scheme
+.. autofunction:: clean_tweet_url
+.. autofunction:: clean_wayback_machine_url
+.. autofunction:: delete_tweet_pathnames
+.. autofunction:: get_response
+.. autofunction:: is_tweet_url
+.. autofunction:: semicolon_parser
+.. autofunction:: timestamp_parser
+
+
+Config
+------------
+
+.. automodule:: waybacktweets.config.config
+    :members:

docs/cli.rst

@@ -0,0 +1,77 @@
+CLI
+================
+
+Usage
+---------
+
+.. click:: waybacktweets._cli:main
+   :prog: waybacktweets
+   :nested: full
+
+Collapsing
+------------
+
+The Wayback Tweets command line tool recommends the use of three types of "collapse": ``urlkey``, ``digest``, and ``timestamp`` field.
+
+- ``urlkey``: (`str`) A canonical transformation of the URL you supplied, for example, ``org,eserver,tc)/``. Such keys are useful for indexing.
+
+- ``digest``: (`str`) The ``SHA1`` hash digest of the content, excluding the headers. It's usually a base-32-encoded string.
+
+- ``timestamp``: (`datetime`) A 14 digit date-time representation in the ``YYYYMMDDhhmmss`` format. We recommend ``YYYYMMDD``.
+
+However, it is possible to use it with other options. Read below text extracted from the official Wayback CDX Server API (Beta) documentation.
+
+.. note::
+
+   A new form of filtering is the option to "collapse" results based on a field, or a substring of a field. Collapsing is done on adjacent CDX lines where all captures after the first one that are duplicate are filtered out. This is useful for filtering out captures that are "too dense" or when looking for unique captures.
+
+   To use collapsing, add one or more ``collapse=field`` or ``collapse=field:N`` where ``N`` is the first ``N`` characters of field to test.
+
+   - Ex: Only show at most 1 capture per hour (compare the first 10 digits of the ``timestamp`` field). Given 2 captures ``20130226010000`` and ``20130226010800``, since first 10 digits ``2013022601`` match, the 2nd capture will be filtered out:
+
+      http://web.archive.org/cdx/search/cdx?url=google.com&collapse=timestamp:10
+
+      The calendar page at `web.archive.org` uses this filter by default: `http://web.archive.org/web/*/archive.org`
+
+   - Ex: Only show unique captures by ``digest`` (note that only adjacent digest are collapsed, duplicates elsewhere in the cdx are not affected):
+
+      http://web.archive.org/cdx/search/cdx?url=archive.org&collapse=digest
+
+   - Ex: Only show unique urls in a prefix query (filtering out captures except first capture of a given url). This is similar to the old prefix query in wayback (note: this query may be slow at the moment):
+
+      http://web.archive.org/cdx/search/cdx?url=archive.org&collapse=urlkey&matchType=prefix
+
+
+URL Match Scope
+-----------------
+
+The CDX Server can return results matching a certain prefix, a certain host or all subdomains by using the ``matchType`` param.
+
+The package ``waybacktweets`` uses the pathname ``/status`` followed by the wildcard '*' at the end of the URL to retrieve only tweets. However, if a value is provided for this parameter, the search will be made from the URL `twitter.com/<USERNAME>`.
+
+Read below text extracted from the official Wayback CDX Server API (Beta) documentation.
+
+.. note::
+
+   For example, if given the url: archive.org/about/ and:
+
+   - ``matchType=exact`` (default if omitted) will return results matching exactly archive.org/about/
+
+   - ``matchType=prefix`` will return results for all results under the path archive.org/about/
+
+      http://web.archive.org/cdx/search/cdx?url=archive.org/about/&matchType=prefix&limit=1000
+
+   - ``matchType=host`` will return results from host archive.org
+
+      http://web.archive.org/cdx/search/cdx?url=archive.org/about/&matchType=host&limit=1000
+
+   - ``matchType=domain`` will return results from host archive.org and all subhosts \*.archive.org
+
+      http://web.archive.org/cdx/search/cdx?url=archive.org/about/&matchType=domain&limit=1000
+
+   The matchType may also be set implicitly by using wildcard '*' at end or beginning of the url:
+
+   - If url is ends in '/\*', eg url=archive.org/\* the query is equivalent to url=archive.org/&matchType=prefix
+   - If url starts with '\*.', eg url=\*.archive.org/ the query is equivalent to url=archive.org/&matchType=domain
+
+   (Note: The domain mode is only available if the CDX is in `SURT <http://crawler.archive.org/articles/user_manual/glossary.html#surt>`_-order format.)

docs/conf.py

@@ -0,0 +1,52 @@
+import datetime
+
+from pallets_sphinx_themes import ProjectLink, get_version
+
+project = "Wayback Tweets"
+release, version = get_version("waybacktweets")
+rst_epilog = f".. |release| replace:: v{release}"
+copyright = f"2023 - {datetime.datetime.now().year}, Claromes · Icon by The Doodle Library · Title font by Google, licensed under the Open Font License · Pre-release: v{release}"  # noqa: E501
+author = "Claromes"
+
+# -- General configuration ---------------------------------------------------
+
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.extlinks",
+    "sphinx.ext.intersphinx",
+    "pallets_sphinx_themes",
+    "sphinxcontrib.mermaid",
+    "sphinx_new_tab_link",
+    "sphinx_click.ext",
+    "sphinx_autodoc_typehints",
+]
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+autodoc_typehints = "description"
+
+# -- Options for HTML output -------------------------------------------------
+
+html_theme = "flask"
+html_static_path = ["_static"]
+html_css_files = ["css/custom.css"]
+html_context = {
+    "project_links": [
+        ProjectLink("PyPI Releases", "https://pypi.org/project/waybacktweets/"),
+        ProjectLink("Source Code", "https://github.com/claromes/waybacktweets/"),
+        ProjectLink(
+            "Issue Tracker", "https://github.com/claromes/waybacktweets/issues/"
+        ),
+        ProjectLink("Mastodon", "https://ruby.social/@claromes"),
+        ProjectLink("Bluesky", "https://bsky.app/profile/claromes.com"),
+    ]
+}
+html_sidebars = {
+    "index": ["project.html", "localtoc.html", "searchbox.html"],
+    "**": ["localtoc.html", "relations.html", "searchbox.html"],
+}
+html_favicon = "../assets/parthenon.png"
+html_logo = "../assets/parthenon.png"
+html_title = f"Wayback Tweets Documentation ({version})"
+html_show_sourcelink = False

docs/contribute.rst

@@ -0,0 +1,38 @@
+Contribute
+================
+
+Here are all the ways you can contribute to this project.
+
+Testing
+---------
+
+The best way to help is by using the package, either on the command line or as a module, suggesting improvements and reporting bugs. You're very welcome to `open an issue <https://github.com/claromes/waybacktweets/issues/>`_.
+
+
+Hacking
+---------
+
+If you have Python skills, contribute to the `code <https://github.com/claromes/waybacktweets/>`_.
+
+These are the prerequisites:
+
+- Python 3.10+
+- Poetry
+
+Install from the source, following the :ref:`installation` instructions.
+
+Brief explanation about the code under the Wayback Tweets directory:
+
+- ``app``: Streamlit application code
+- ``assets``: Title and logo images
+- ``docs``: Documentation generated with Sphinx
+- ``legacy_app``: Legacy Streamlit application code
+- ``waybacktweets/api``: Main package modules
+- ``waybacktweets/config``: Global configuration module
+- ``waybacktweets/exceptions``: Wayback Tweets Exceptions
+- ``waybacktweets/utils``: Helper functions used in the package
+
+Sponsoring
+------------
+
+You can also donate to the project's developer and maintainer, `Claromes <https://claromes.com>`_, via `GitHub Sponsor <https://github.com/sponsors/claromes>`_ or if you are interested in sponsoring the project you can contact via email at support at claromes dot com.

docs/exceptions.rst

@@ -0,0 +1,44 @@
+Exceptions
+================
+
+These are the most common errors and are handled by the ``waybacktweets`` package.
+
+ReadTimeoutError
+------------------
+
+This error occurs when a request to the web.archive.org server takes too long to respond. The server could be overloaded or there could be network issues.
+
+The output message from the package would be: ``Connection to web.archive.org timed out.``
+
+ConnectionError
+------------------
+
+This error is raised when the package fails to establish a new connection with web.archive.org. This could be due to network issues or the server being down.
+
+The output message from the package would be: ``Failed to establish a new connection with web.archive.org. Max retries exceeded.``
+
+
+This is the error often returned when performing experimental parsing of URLs with the mimetype ``application/json``.
+
+The warning output message from the package would be: ``Connection error with https://web.archive.org/web/<TIMESTAMP>/https://twitter.com/<USERNAME>/status/<TWEET_ID>. Max retries exceeded. Error parsing the JSON, but the CDX data was saved.``
+
+HTTPError
+------------------
+
+This error occurs when the Internet Archive services are temporarily offline. This could be due to maintenance or server issues.
+
+The output message from the package would be: ``Temporarily Offline: Internet Archive services are temporarily offline. Please check Internet Archive Twitter feed (https://twitter.com/internetarchive) for the latest information.``
+
+EmptyResponseError
+---------------------
+
+This exception raised for empty responses.
+
+The output message from the package would be: ``No data was saved due to an empty response.``
+
+Warning
+------------------
+
+It is possible to encounter the following warning when running the ``TweetsParser`` class (:ref:`parser`): ``<TWEET_URL> not available on the user's Twitter account, but the CDX data was saved.``
+
+This occurs when the original tweet is no longer available on Twitter and has possibly been deleted.

docs/field_options.rst

@@ -0,0 +1,42 @@
+.. _field_options:
+
+Field Options
+================
+
+The package performs several parses to facilitate the analysis of archived tweets and types of tweets. The fields below are available, which can be passed to the :ref:`parser` and :ref:`exporter`, in addition, the command line tool returns all these fields.
+
+- ``archived_urlkey``: (`str`) A canonical transformation of the URL you supplied, for example, ``org,eserver,tc)/``. Such keys are useful for indexing.
+
+- ``archived_timestamp``: (`str`) A 14 digit date-time representation in the ``YYYYMMDDhhmmss`` format.
+
+- ``parsed_archived_timestamp``: (`str`) The ``archived_timestamp`` in human-readable format.
+
+- ``archived_tweet_url``: (`str`) The archived URL.
+
+- ``parsed_archived_tweet_url``: (`str`) The archived URL after parsing. It is not guaranteed that this option will be archived, it is just a facilitator, as the originally archived URL does not always exist, due to changes in URLs and web services of the social network Twitter. Check the :ref:`utils`.
+
+- ``original_tweet_url``: (`str`) The original tweet URL.
+
+- ``parsed_tweet_url``: (`str`) The original tweet URL after parsing. Old URLs were archived in a nested manner. The parsing applied here unnests these URLs, when necessary.  Check the :ref:`utils`.
+
+- ``available_tweet_text``: (`str`) The tweet text extracted from the URL that is still available on the Twitter account.
+
+- ``available_tweet_is_RT``: (`bool`) Whether the tweet from the ``available_tweet_text`` field is a retweet or not.
+
+- ``available_tweet_info``: (`str`) Name and date of the tweet from the ``available_tweet_text`` field.
+
+- ``archived_mimetype``: (`str`) The mimetype of the archived content, which can be one of these:
+
+    - ``text/html``
+
+    - ``warc/revisit``
+
+    - ``application/json``
+
+    - ``unk``
+
+- ``archived_statuscode``: (`str`) The HTTP status code of the snapshot. If the mimetype is ``warc/revisit``, the value returned for the ``statuscode`` key can be blank, but the actual value is the same as that of any other entry that has the same ``digest`` as this entry. If the mimetype is ``application/json``, the value is usually empty or ``-``.
+
+- ``archived_digest``: (`str`) The ``SHA1`` hash digest of the content, excluding the headers. It's usually a base-32-encoded string.
+
+- ``archived_length``: (`int`) The compressed byte size of the corresponding WARC record, which includes WARC headers, HTTP headers, and content payload.

docs/index.rst

@@ -0,0 +1,83 @@
+.. rst-class:: hide-header
+
+Wayback Tweets
+================
+
+.. image:: ../assets/waybacktweets_title.png
+    :alt: Wayback Tweets
+    :align: center
+
+Pre-release: |release|
+
+Retrieves archived tweets CDX data from the Wayback Machine, performs necessary parsing (see :ref:`field_options`), and saves the data in HTML (for easy viewing of the tweets using the ``iframe`` tag), CSV, and JSON formats.
+
+.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.12528448.svg
+  :target: https://doi.org/10.5281/zenodo.12528448
+
+.. note::
+    Intensive queries can lead to rate limiting, resulting in a temporary ban of a few minutes from web.archive.org.
+
+
+User Guide
+------------
+
+.. toctree::
+    :maxdepth: 2
+
+    installation
+    quickstart
+    workflow
+    field_options
+    outputs
+    exceptions
+    contribute
+    todo
+
+
+Command-Line Interface
+------------------------
+.. toctree::
+    :maxdepth: 2
+
+    cli
+
+Streamlit Web App
+-------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    streamlit
+
+
+API Reference
+---------------
+
+.. toctree::
+    :maxdepth: 2
+
+    api
+
+
+Additional Information
+-----------------------
+
+.. toctree::
+    :maxdepth: 1
+
+.. raw:: html
+
+    <ul>
+        <li><a href="https://github.com/claromes/waybacktweets/blob/main/LICENSE.md" target="_blank">GPL-3.0 license</a></li>
+        <li><a href="https://github.com/claromes/waybacktweets/releases" target="_blank">Changes</a></li>
+    </ul>
+
+Indices and tables
+----------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    genindex
+    modindex
+    search

docs/installation.rst

@@ -0,0 +1,64 @@
+.. _installation:
+
+Installation
+================
+
+
+Using pip
+------------
+
+    .. code-block:: shell
+
+        pip install waybacktweets
+
+From source
+-------------
+
+    Clone the repository:
+
+    .. code-block:: shell
+
+        git clone git@github.com:claromes/waybacktweets.git
+
+    Change directory:
+
+    .. code-block:: shell
+
+        cd waybacktweets
+
+    Install poetry, if you haven't already:
+
+    .. code-block:: shell
+
+        pip install poetry
+
+
+    Install the dependencies:
+
+    .. code-block:: shell
+
+        poetry install
+
+    Run the CLI:
+
+    .. code-block:: shell
+
+        poetry run waybacktweets [SUBCOMMANDS]
+
+    Run the Streamlit App:
+
+    .. code-block:: shell
+
+        streamlit run app/app.py
+
+    Build the docs:
+
+    .. code-block:: shell
+
+        cd docs
+
+    .. code-block:: shell
+
+        make clean html
+
+`Read the Poetry CLI documentation <https://python-poetry.org/docs/cli/>`_.

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/outputs.rst

@@ -0,0 +1,29 @@
+Outputs
+==========
+
+It is possible to save the CDX data in three formats. In the command line tool, these three formats are saved automatically.
+
+HTML
+--------
+
+This format allows for easy viewing of the archived tweets, through the use of the ``iframe`` tag. Each tweet contains four viewing options, which render when clicking on the accordion:
+
+- ``archived_tweet_url``: (`str`) The archived URL.
+
+- ``parsed_archived_tweet_url``: (`str`) The archived URL after parsing. It is not guaranteed that this option will be archived, it is just a facilitator, as the originally archived URL does not always exist, due to changes in URLs and web services of the social network Twitter. Check the :ref:`utils`.
+
+- ``original_tweet_url``: (`str`) The original tweet URL.
+
+- ``parsed_tweet_url``: (`str`) The original tweet URL after parsing. Old URLs were archived in a nested manner. The parsing applied here unnests these URLs, when necessary.  Check the :ref:`utils`.
+
+Additionally, other fields are displayed.
+
+CSV
+--------
+
+Option to analyze the CDX data in comma-separated values.
+
+JSON
+--------
+
+Option to analyze the data in JavaScript Object Notation.

docs/quickstart.rst

@@ -0,0 +1,48 @@
+Quickstart
+================
+
+CLI
+-------------
+
+Using Wayback Tweets as a standalone command line tool.
+
+waybacktweets [OPTIONS] USERNAME
+
+.. code-block:: shell
+
+    waybacktweets --from 20150101 --to 20191231 --limit 250 jack
+
+Web App
+-------------
+
+Using Wayback Tweets as a Streamlit Web App.
+
+`Open the application <https://waybacktweets.streamlit.app>`_, a prototype written in Python with the Streamlit framework and hosted on Streamlit Cloud.
+
+Module
+-------------
+
+Using Wayback Tweets as a Python Module.
+
+.. code-block:: python
+
+    from waybacktweets import WaybackTweets, TweetsParser, TweetsExporter
+
+    USERNAME = "jack"
+
+    api = WaybackTweets(USERNAME)
+    archived_tweets = api.get()
+
+    if archived_tweets:
+        field_options = [
+            "archived_timestamp",
+            "original_tweet_url",
+            "archived_tweet_url",
+            "archived_statuscode",
+        ]
+
+        parser = TweetsParser(archived_tweets, USERNAME, field_options)
+        parsed_tweets = parser.parse()
+
+        exporter = TweetsExporter(parsed_tweets, USERNAME, field_options)
+        exporter.save_to_csv()

docs/streamlit.rst

@@ -0,0 +1,63 @@
+Web App
+=========
+
+The application is a prototype hosted on Streamlit Cloud, serving as an alternative to the command line tool.
+
+`Open the application <https://waybacktweets.streamlit.app>`_.
+
+
+Filters
+----------
+
+- Filtering by date range: Using the ``from`` and ``to`` filters
+
+- Limit: Query result limits.
+
+- Offset: Allows for a simple way to scroll through the results.
+
+- Only unique Wayback Machine URLs: Filtering by the collapse option using the ``urlkey`` field and the URL Match Scope ``prefix``
+
+
+Username Query Parameter
+--------------------------
+
+An alternative way to access the application is by using the ``username`` query parameter. This allows for automatic configuration of the Username input and automatically searches. Additionally, when the ``username`` parameter is sent, the accordion with the filters will already be open.
+
+Example URL format:
+
+``https://waybacktweets.streamlit.app?username=<USERNAME>``
+
+
+Community Comments
+--------------------
+
+.. raw:: html
+
+   <ul>
+        <li>"We're always delighted when we see our community members create tools for open source research." <a href="https://twitter.com/bellingcat/status/1728085974138122604" target="_blank">Bellingcat</a></li>
+        <br>
+        <li>"#myOSINTtip Clarissa Mendes launched a new tool for accessing old tweets via archive.org called the Wayback Tweets app. For those who love to look deeper at #osint tools, it is available on GitHub and uses the Wayback CDX Server API server (which is a hidden gem for accessing archive.org data!)" <a href="https://www.linkedin.com/posts/my-osint-training_myosinttip-osint-activity-7148425933324963841-0Q2n/" target="_blank">My OSINT Training</a></li>
+        <br>
+        <li>"Original way to find deleted tweets." <a href="https://twitter.com/henkvaness/status/1693298101765701676" target="_blank">Henk Van Ess</a></li>
+        <br>
+        <li>"This is an excellent tool to use now that most Twitter API-based tools have gone down with changes to the pricing structure over at X." <a href="https://osintnewsletter.com/p/22#%C2%A7osint-community" target="_blank">The OSINT Newsletter - Issue #22</a></li>
+        <br>
+        <li>"One of the keys to using the Wayback Machine effectively is knowing what it can and can't archive. It can, and has, archived many, many Twitter accounts... Utilize fun tools such as Wayback Tweets to do so more effectively." <a href="https://memeticwarfareweekly.substack.com/p/mww-paradise-by-the-telegram-dashboard" target="_blank">Ari Ben Am</a></li>
+        <br>
+        <li>"Want to see archived tweets on Wayback Machine in bulk? You can use Wayback Tweets." <a href="https://twitter.com/DailyOsint/status/1695065018662855102" target="_blank">Daily OSINT</a></li>
+        <br>
+        <li>"Untuk mempermudah penelusuran arsip, gunakan Wayback Tweets." <a href="https://twitter.com/gijnIndonesia/status/1685912219408805888" target="_blank">GIJN Indonesia</a></li>
+        <br>
+        <li>"A tool to quickly view tweets saved on archive.org." <a href="https://irinatechtips.substack.com/p/irina_tech_tips-newsletter-3-2023#%C2%A7wayback-tweets" target="_blank">Irina_Tech_Tips Newsletter #3</a></li>
+        <br>
+    </ul>
+
+Legacy App
+-------------
+
+To access the legacy version of Wayback Tweets `click here <https://waybacktweets-legacy.streamlit.app>`_.
+
+.. note::
+
+   If the application is down, please check the `Streamlit Cloud Status <https://www.streamlitstatus.com/>`_.
+

docs/todo.rst

@@ -0,0 +1,18 @@
+TODO
+================
+
+.. |uncheck| raw:: html
+
+    <input type="checkbox">
+
+|uncheck| Unit Tests
+
+|uncheck| JSON Parser: Create a separate function to handle JSON return, apply JsonParser (``waybacktweets/api/parse.py:110``), and avoid rate limiting
+
+|uncheck| Download images when tweet URL has extensions like JPG or PNG
+
+|uncheck| Implement logging system (remove print statements)
+
+|uncheck| Mapping and parsing of other Twitter-related URLs
+
+|uncheck| Develop a scraper to download snapshots from https://archive.today

docs/workflow.rst

@@ -0,0 +1,25 @@
+.. _flowchart:
+
+Workflow
+================
+
+The tool was written following a proposal not only to retrieve data from archived tweets, but also to facilitate the reading of these tweets. Therefore, a flow is defined to obtain these results in the best possible way.
+
+Due to limitations of the Wayback CDX Server API, it is not always possible to parse the results with the mimetype ``application/json``, regardless, the data in CDX format are saved.
+
+Use the mouse to zoom in and out the flowchart.
+
+.. mermaid::
+   :zoom:
+   :align: center
+
+   flowchart TB
+      A[input Username]--> B[(Wayback Machine)]
+      B--> B1[save Archived Tweets CDX data]
+      B1--> |parsing| C{embed Tweet URL\nvia Twitter Publisher}
+      C--> |2xx/3xx| D[return Tweet text]
+      C--> |4xx| E[return None]
+      E--> F{request Archived\nTweet URL}
+      F--> |4xx| G[return Only CDX data]
+      F--> |TODO: 2xx/3xx: application/json| J[return JSON text]
+      F--> |2xx/3xx: text/html, warc/revisit, unk| K[return HTML iframe tag]

legacy_app/legacy_app.py

@@ -0,0 +1,525 @@
+import datetime
+import re
+from urllib.parse import unquote
+
+import requests
+import streamlit as st
+import streamlit.components.v1 as components
+
+year = datetime.datetime.now().year
+
+st.set_page_config(
+    page_title="Wayback Tweets",
+    page_icon="🏛️",
+    layout="centered",
+    menu_items={
+        "About": """
+        ## 🏛️ Wayback Tweets
+
+        Tool that displays, via Wayback CDX Server API, multiple archived tweets on Wayback Machine to avoid opening each link manually. Users can apply filters based on specific years and view tweets that do not have the original URL available.
+
+        This tool is a prototype, please feel free to send your [feedbacks](https://github.com/claromes/waybacktweets/issues). Created by [@claromes](https://claromes.com).
+
+        -------
+        """,  # noqa: E501
+    },
+)
+
+# https://discuss.streamlit.io/t/remove-hide-running-man-animation-on-top-of-page/21773/3
+hide_streamlit_style = """
+<style>
+    header[data-testid="stHeader"] {
+        opacity: 0.5;
+    }
+     iframe {
+        border: 1px solid #dddddd;
+        border-radius: 0.5rem;
+    }
+    div[data-testid="InputInstructions"] {
+        visibility: hidden;
+    }
+</style>
+"""
+
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+if "current_handle" not in st.session_state:
+    st.session_state.current_handle = ""
+
+if "prev_disabled" not in st.session_state:
+    st.session_state.prev_disabled = False
+
+if "next_disabled" not in st.session_state:
+    st.session_state.next_disabled = False
+
+if "next_button" not in st.session_state:
+    st.session_state.next_button = False
+
+if "prev_button" not in st.session_state:
+    st.session_state.prev_button = False
+
+if "update_component" not in st.session_state:
+    st.session_state.update_component = 0
+
+if "offset" not in st.session_state:
+    st.session_state.offset = 0
+
+if "saved_at" not in st.session_state:
+    st.session_state.saved_at = (2006, year)
+
+if "count" not in st.session_state:
+    st.session_state.count = False
+
+
+def scroll_into_view():
+    js = f"""
+    <script>
+        window.parent.document.querySelector('section.main').scrollTo(0, 0);
+        let update_component = {st.session_state.update_component} // Force component update to generate scroll
+    </script>
+    """  # noqa: E501
+
+    components.html(js, width=0, height=0)
+
+
+def clean_tweet(tweet):
+    handle = st.session_state.current_handle.lower()
+    tweet_lower = tweet.lower()
+
+    pattern = re.compile(r"/status/(\d+)")
+    match_lower_case = pattern.search(tweet_lower)
+    match_original_case = pattern.search(tweet)
+
+    if match_lower_case and handle in tweet_lower:
+        return f"https://twitter.com/{st.session_state.current_handle}/status/{match_original_case.group(1)}"  # noqa: E501
+    else:
+        return tweet
+
+
+def clean_link(link):
+    handle = st.session_state.current_handle.lower()
+    link = link.lower()
+
+    pattern = re.compile(r"/status/(\d+)")
+    match = pattern.search(link)
+
+    if match and handle in link:
+        return f"https://web.archive.org/web/{timestamp[i]}/https://twitter.com/{st.session_state.current_handle}/status/{match.group(1)}"  # noqa: E501
+    else:
+        return link
+
+
+def pattern_tweet(tweet):
+    # Reply: /status//
+    # Link:  /status///
+    # Twimg: /status/https://pbs
+
+    pattern = re.compile(r'/status/"([^"]+)"')
+
+    match = pattern.search(tweet)
+    if match:
+        return match.group(1).lstrip("/")
+    else:
+        return tweet
+
+
+def pattern_tweet_id(tweet):
+    # Delete sub-endpoint (/photos, /likes, /retweet...)
+    pattern_username = re.compile(r"https://twitter\.com/([^/]+)/status/\d+")
+    match_username = pattern_username.match(tweet)
+
+    pattern_id = r"https://twitter.com/\w+/status/(\d+)"
+    match_id = re.search(pattern_id, tweet)
+
+    if match_id and match_username:
+        tweet_id = match_id.group(1)
+        username = match_username.group(1)
+        return f"https://twitter.com/{username}/status/{tweet_id}"
+    else:
+        return tweet
+
+
+def check_double_status(url_wb, url_tweet):
+    if url_wb.count("/status/") == 2 and "twitter.com" not in url_tweet:
+        return True
+
+    return False
+
+
+def embed(tweet):
+    try:
+        url = f"https://publish.twitter.com/oembed?url={clean_tweet(tweet)}"
+        response = requests.get(url)
+
+        regex = r'<blockquote class="twitter-tweet"(?: [^>]+)?><p[^>]*>(.*?)<\/p>.*?&mdash; (.*?)<\/a>'  # noqa: E501
+        regex_author = r"^(.*?)\s*\("
+
+        if response.status_code == 200 or response.status_code == 302:
+            status_code = response.status_code
+            html = response.json()["html"]
+            author_name = response.json()["author_name"]
+
+            matches_html = re.findall(regex, html, re.DOTALL)
+
+            tweet_content = []
+            user_info = []
+            is_RT = []
+
+            for match in matches_html:
+                tweet_content_match = re.sub(r"<a[^>]*>|<\/a>", "", match[0].strip())
+                tweet_content_match = tweet_content_match.replace("<br>", "\n")
+
+                user_info_match = re.sub(r"<a[^>]*>|<\/a>", "", match[1].strip())
+                user_info_match = user_info_match.replace(")", "), ")
+
+                match_author = re.search(regex_author, user_info_match)
+                author_tweet = match_author.group(1)
+
+                if tweet_content_match:
+                    tweet_content.append(tweet_content_match)
+                if user_info_match:
+                    user_info.append(user_info_match)
+
+                    is_RT_match = False
+                    if author_name != author_tweet:
+                        is_RT_match = True
+
+                    is_RT.append(is_RT_match)
+
+            return status_code, tweet_content, user_info, is_RT
+        else:
+            return False
+    except requests.exceptions.Timeout:
+        st.error("Connection to web.archive.org timed out.")
+    except requests.exceptions.ConnectionError:
+        st.error("Failed to establish a new connection with web.archive.org.")
+    except UnboundLocalError:
+        st.empty()
+
+
+@st.cache_data(ttl=1800, show_spinner=False)
+def tweets_count(handle, saved_at):
+    url = f"https://web.archive.org/cdx/search/cdx?url=https://twitter.com/{handle}/status/*&collapse=timestamp:8&output=json&from={saved_at[0]}&to={saved_at[1]}"  # noqa: E501
+    try:
+        response = requests.get(url)
+
+        if response.status_code == 200:
+            data = response.json()
+            if data and len(data) > 1:
+                total_tweets = len(data) - 1
+                return total_tweets
+            else:
+                return 0
+    except requests.exceptions.Timeout:
+        st.error("Connection to web.archive.org timed out.")
+        st.stop()
+    except requests.exceptions.ConnectionError:
+        st.error("Failed to establish a new connection with web.archive.org.")
+    except UnboundLocalError:
+        st.empty()
+
+
+@st.cache_data(ttl=1800, show_spinner=False)
+def query_api(handle, limit, offset, saved_at):
+    if not handle:
+        st.warning("username, please!")
+        st.stop()
+
+    url = f"https://web.archive.org/cdx/search/cdx?url=https://twitter.com/{handle}/status/*&collapse=timestamp:8&output=json&limit={limit}&offset={offset}&from={saved_at[0]}&to={saved_at[1]}"  # noqa: E501
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+
+        if response.status_code == 200 or response.status_code == 304:
+            return response.json()
+    except requests.exceptions.Timeout:
+        st.error("Connection to web.archive.org timed out.")
+    except requests.exceptions.ConnectionError:
+        st.error("Failed to establish a new connection with web.archive.org.")
+    except UnboundLocalError:
+        st.empty()
+    except requests.exceptions.HTTPError:
+        st.error(
+            """
+        **Temporarily Offline**
+
+        Internet Archive services are temporarily offline. Please check Internet Archive [Twitter feed](https://twitter.com/internetarchive/) for the latest information.
+        """  # noqa: E501
+        )
+        st.stop()
+
+
+@st.cache_data(ttl=1800, show_spinner=False)
+def parse_links(links):
+    parsed_links = []
+    timestamp = []
+    tweet_links = []
+    parsed_mimetype = []
+
+    for link in links[1:]:
+        tweet_remove_char = unquote(link[2]).replace("’", "")
+        cleaned_tweet = pattern_tweet(tweet_remove_char).strip('"')
+
+        url = f"https://web.archive.org/web/{link[1]}/{tweet_remove_char}"
+
+        parsed_links.append(url)
+        timestamp.append(link[1])
+        tweet_links.append(cleaned_tweet)
+        parsed_mimetype.append(link[3])
+
+    return parsed_links, tweet_links, parsed_mimetype, timestamp
+
+
+def attr(i):
+    original_tweet = pattern_tweet_id(clean_tweet(tweet_links[i]))
+
+    if status:
+        original_tweet = pattern_tweet_id(f"https://twitter.com/{tweet_links[i]}")
+    elif "://" not in tweet_links[i]:
+        original_tweet = pattern_tweet_id(f"https://{tweet_links[i]}")
+
+    st.markdown(
+        f'{i+1 + st.session_state.offset}. [**archived url**]({link}) · [**original url**]({original_tweet}) · **MIME Type:** {mimetype[i]} · **Saved at:** {datetime.datetime.strptime(timestamp[i], "%Y%m%d%H%M%S")}'  # noqa: E501
+    )
+
+
+def display_tweet():
+    if (
+        mimetype[i] == "application/json"
+        or mimetype[i] == "text/html"
+        or mimetype[i] == "unk"
+        or mimetype[i] == "warc/revisit"
+    ):
+        if is_RT[0] is True:
+            st.info("*Retweet*")
+        st.write(tweet_content[0])
+        st.write(f"**{user_info[0]}**")
+
+        st.divider()
+    else:
+        st.warning("MIME Type was not parsed.")
+
+        st.divider()
+
+
+def display_not_tweet():
+    original_link = pattern_tweet_id(clean_tweet(tweet_links[i]))
+
+    if status:
+        original_link = pattern_tweet_id(f"https://twitter.com/{tweet_links[i]}")
+    elif "://" not in tweet_links[i]:
+        original_link = pattern_tweet_id(f"https://{tweet_links[i]}")
+
+    response_html = requests.get(original_link)
+
+    if (
+        mimetype[i] == "text/html"
+        or mimetype[i] == "warc/revisit"
+        or mimetype[i] == "unk"
+    ):
+        if (
+            ".jpg" in tweet_links[i] or ".png" in tweet_links[i]
+        ) and response_html.status_code == 200:
+            components.iframe(tweet_links[i], height=500, scrolling=True)
+        elif "/status/" not in original_link:
+            st.info("This isn't a status or is not available")
+        elif status or f"{st.session_state.current_handle}" not in original_link:
+            st.info(f"Replying to {st.session_state.current_handle}")
+        else:
+            components.iframe(clean_link(link), height=500, scrolling=True)
+
+        st.divider()
+    elif mimetype[i] == "application/json":
+        try:
+            response_json = requests.get(link)
+
+            if response_json.status_code == 200:
+                json_data = response_json.json()
+
+                if "data" in json_data:
+                    if "text" in json_data["data"]:
+                        json_text = json_data["data"]["text"]
+                    else:
+                        json_text = json_data["data"]
+                else:
+                    if "text" in json_data:
+                        json_text = json_data["text"]
+                    else:
+                        json_text = json_data
+
+                st.code(json_text)
+                st.json(json_data, expanded=False)
+
+                st.divider()
+            else:
+                st.error(response_json.status_code)
+
+                st.divider()
+        except requests.exceptions.Timeout:
+            st.error("Connection to web.archive.org timed out.")
+            st.divider()
+        except requests.exceptions.ConnectionError:
+            st.error("Failed to establish a new connection with web.archive.org.")
+            st.divider()
+        except UnboundLocalError:
+            st.empty()
+    else:
+        st.warning("MIME Type was not parsed.")
+        st.divider()
+
+
+def prev_page():
+    st.session_state.offset -= tweets_per_page
+
+    # scroll to top config
+    st.session_state.update_component += 1
+    scroll_into_view()
+
+
+def next_page():
+    st.session_state.offset += tweets_per_page
+
+    # scroll to top config
+    st.session_state.update_component += 1
+    scroll_into_view()
+
+
+# UI
+st.title(
+    "Wayback Tweets [![Star](https://img.shields.io/github/stars/claromes/waybacktweets?style=social)](https://github.com/claromes/waybacktweets)",  # noqa: E501
+    anchor=False,
+    help="v0.4.3",
+)
+st.write(
+    "Display multiple archived tweets on Wayback Machine and avoid opening each link manually"  # noqa: E501
+)
+
+handle = st.text_input("Username", placeholder="jack")
+
+st.session_state.saved_at = st.slider("Tweets saved between", 2006, year, (2006, year))
+
+not_available = st.checkbox(
+    "Original URLs not available",
+    help="Due to changes in X, it is possible to find available tweets if you are logged into X",  # noqa: E501
+)
+
+query = st.button("Query", type="primary", use_container_width=True)
+
+if handle != st.session_state.current_handle:
+    st.session_state.current_handle = handle
+    st.session_state.offset = 0
+
+if query or st.session_state.count:
+    tweets_per_page = 25
+
+    st.session_state.count = tweets_count(handle, st.session_state.saved_at)
+
+    st.caption(
+        "The search optimization uses an 8-digit [collapsing strategy](https://github.com/internetarchive/wayback/blob/master/wayback-cdx-server/README.md?ref=hackernoon.com#collapsing), refining the captures to one per day. The number of tweets per page is set to 25, and this is a fixed value due to the API rate limit."  # noqa: E501
+    )
+    st.write(f"**{st.session_state.count} URLs have been captured**")
+
+    if st.session_state.count:
+        if tweets_per_page > st.session_state.count:
+            tweets_per_page = st.session_state.count
+
+    try:
+        progress = st.empty()
+        links = query_api(
+            handle, tweets_per_page, st.session_state.offset, st.session_state.saved_at
+        )
+
+        parse = parse_links(links)
+        parsed_links = parse[0]
+        tweet_links = parse[1]
+        mimetype = parse[2]
+        timestamp = parse[3]
+
+        if links:
+            st.divider()
+
+            st.session_state.current_handle = handle
+
+            return_none_count = 0
+
+            start_index = st.session_state.offset
+            end_index = min(st.session_state.count, start_index + tweets_per_page)
+
+            with st.spinner("Fetching..."):
+                for i in range(tweets_per_page):
+                    try:
+                        if tweet_links[i]:
+                            link = parsed_links[i]
+                            tweet = embed(tweet_links[i])
+
+                            status = check_double_status(link, tweet_links[i])
+
+                            if not not_available:
+                                attr(i)
+
+                                if tweet:
+                                    status_code = tweet[0]
+                                    tweet_content = tweet[1]
+                                    user_info = tweet[2]
+                                    is_RT = tweet[3]
+
+                                    display_tweet()
+                                elif not tweet:
+                                    display_not_tweet()
+
+                            if not_available:
+                                if not tweet:
+                                    return_none_count += 1
+                                    attr(i)
+
+                                    display_not_tweet()
+
+                                progress.write(
+                                    f"{return_none_count} URLs have been captured in the range {start_index}-{end_index}"  # noqa: E501
+                                )
+
+                            if start_index <= 0:
+                                st.session_state.prev_disabled = True
+                            else:
+                                st.session_state.prev_disabled = False
+
+                            if i + 1 == st.session_state.count:
+                                st.session_state.next_disabled = True
+                            else:
+                                st.session_state.next_disabled = False
+                    except IndexError:
+                        if start_index <= 0:
+                            st.session_state.prev_disabled = True
+                        else:
+                            st.session_state.prev_disabled = False
+
+                        st.session_state.next_disabled = True
+
+            prev, _, next = st.columns([3, 4, 3])
+
+            prev.button(
+                "Previous",
+                disabled=st.session_state.prev_disabled,
+                key="prev_button_key",
+                on_click=prev_page,
+                type="primary",
+                use_container_width=True,
+            )
+            next.button(
+                "Next",
+                disabled=st.session_state.next_disabled,
+                key="next_button_key",
+                on_click=next_page,
+                type="primary",
+                use_container_width=True,
+            )
+
+        if not links:
+            st.error("Unable to query the Wayback Machine API.")
+    except TypeError as e:
+        st.error(
+            f"""
+        {e}. Refresh this page and try again.
+        """  # noqa: E501
+        )
+        st.session_state.offset = 0

legacy_app/requirements.txt

@@ -0,0 +1,2 @@
+requests==2.30.0
+streamlit==1.27.0

waybacktweets/__init__.py

@@ -0,0 +1,6 @@
+# flake8: noqa: F401
+
+from waybacktweets.api.export import TweetsExporter
+from waybacktweets.api.parse import JsonParser, TweetsParser, TwitterEmbed
+from waybacktweets.api.request import WaybackTweets
+from waybacktweets.api.visualize import HTMLTweetsVisualizer

waybacktweets/_cli.py

@@ -0,0 +1,158 @@
+"""
+CLI functions for retrieving archived tweets.
+"""
+
+from datetime import datetime
+from typing import Any, Optional
+
+import click
+from rich import print as rprint
+
+from waybacktweets.api.export import TweetsExporter
+from waybacktweets.api.parse import TweetsParser
+from waybacktweets.api.request import WaybackTweets
+from waybacktweets.config.config import config
+
+
+def _parse_date(
+    ctx: Optional[Any] = None, param: Optional[Any] = None, value: Optional[str] = None
+) -> Optional[str]:
+    """
+    Parses a date string and returns it in the format "YYYYMMDD".
+
+    Args:
+        ctx: Necessary when used with the click package. Defaults to None.
+        param: Necessary when used with the click package. Defaults to None.
+        value: A date string in the "YYYYMMDD" format. Defaults to None.
+
+    Returns:
+        The input date string formatted in the "YYYYMMDD" format, or None if no date string was provided.
+    """  # noqa: E501
+    try:
+        if value is None:
+            return None
+
+        date = datetime.strptime(value, "%Y%m%d")
+
+        return date.strftime("%Y%m%d")
+    except ValueError:
+        raise click.BadParameter("Date must be in format YYYYmmdd")
+
+
+@click.command()
+@click.argument("username", type=str)
+@click.option(
+    "-c",
+    "--collapse",
+    type=click.Choice(["urlkey", "digest", "timestamp:XX"], case_sensitive=False),
+    default=None,
+    help="Collapse results based on a field, or a substring of a field. XX in the timestamp value ranges from 1 to 14, comparing the first XX digits of the timestamp field. It is recommended to use from 4 onwards, to compare at least by years.",  # noqa: E501
+)
+@click.option(
+    "-f",
+    "--from",
+    "timestamp_from",
+    type=click.UNPROCESSED,
+    metavar="DATE",
+    callback=_parse_date,
+    default=None,
+    help="Filtering by date range from this date. Format: YYYYmmdd",
+)
+@click.option(
+    "-t",
+    "--to",
+    "timestamp_to",
+    type=click.UNPROCESSED,
+    metavar="DATE",
+    callback=_parse_date,
+    default=None,
+    help="Filtering by date range up to this date. Format: YYYYmmdd",
+)
+@click.option(
+    "-l",
+    "--limit",
+    type=int,
+    metavar="INTEGER",
+    default=None,
+    help="Query result limits.",
+)
+@click.option(
+    "-o",
+    "--offset",
+    type=int,
+    metavar="INTEGER",
+    default=None,
+    help="Allows for a simple way to scroll through the results.",
+)
+@click.option(
+    "-mt",
+    "--matchtype",
+    type=click.Choice(["exact", "prefix", "host", "domain"], case_sensitive=False),
+    default=None,
+    help="Results matching a certain prefix, a certain host or all subdomains.",  # noqa: E501
+)
+@click.option(
+    "-v",
+    "--verbose",
+    "verbose",
+    is_flag=True,
+    default=False,
+    help="Shows the error log.",
+)
+def main(
+    username: str,
+    collapse: Optional[str],
+    timestamp_from: Optional[str],
+    timestamp_to: Optional[str],
+    limit: Optional[int],
+    offset: Optional[int],
+    matchtype: Optional[str],
+    verbose: Optional[bool],
+) -> None:
+    """
+    Retrieves archived tweets CDX data from the Wayback Machine, performs necessary parsing, and saves the data.
+
+    USERNAME: The Twitter username without @.
+    """  # noqa: E501
+    try:
+        config.verbose = verbose
+
+        api = WaybackTweets(
+            username, collapse, timestamp_from, timestamp_to, limit, offset, matchtype
+        )
+
+        print(f"Waybacking @{username}'s archived tweets...")
+        archived_tweets = api.get()
+
+        if archived_tweets:
+            field_options = [
+                "archived_urlkey",
+                "archived_timestamp",
+                "parsed_archived_timestamp",
+                "archived_tweet_url",
+                "parsed_archived_tweet_url",
+                "original_tweet_url",
+                "parsed_tweet_url",
+                "available_tweet_text",
+                "available_tweet_is_RT",
+                "available_tweet_info",
+                "archived_mimetype",
+                "archived_statuscode",
+                "archived_digest",
+                "archived_length",
+            ]
+
+            parser = TweetsParser(archived_tweets, username, field_options)
+            parsed_tweets = parser.parse(print_progress=True)
+
+            exporter = TweetsExporter(parsed_tweets, username, field_options)
+
+            exporter.save_to_csv()
+            exporter.save_to_json()
+            exporter.save_to_html()
+    except Exception as e:
+        rprint(f"[red]{e}")
+    finally:
+        rprint(
+            "[yellow]\nNeed help? Read the docs: https://claromes.github.io/waybacktweets"  # noqa: E501
+        )

waybacktweets/api/export.py

@@ -0,0 +1,121 @@
+"""
+Exports the parsed archived tweets.
+"""
+
+import datetime
+import os
+import re
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+from waybacktweets.api.visualize import HTMLTweetsVisualizer
+
+
+class TweetsExporter:
+    """
+    Class responsible for exporting parsed archived tweets.
+
+    Args:
+        data (Dict[str, List[Any]]): The parsed archived tweets data.
+        username (str): The username associated with the tweets.
+        field_options (List[str]): The fields to be included in the exported data. For more details on each option, visit :ref:`field_options`.
+    """  # noqa: E501
+
+    def __init__(
+        self, data: Dict[str, List[Any]], username: str, field_options: List[str]
+    ):
+        self.data = data
+        self.username = username
+        self.field_options = field_options
+        self.formatted_datetime = self._datetime_now()
+        self.filename = f"{self.username}_tweets_{self.formatted_datetime}"
+        self.dataframe = self._create_dataframe()
+
+    @staticmethod
+    def _datetime_now() -> str:
+        """
+        Returns the current datetime, formatted as a string.
+
+        Returns:
+            The current datetime.
+        """
+        now = datetime.datetime.now()
+        formatted_now = now.strftime("%Y%m%d%H%M%S")
+        formatted_now = re.sub(r"\W+", "", formatted_now)
+
+        return formatted_now
+
+    @staticmethod
+    def _transpose_matrix(
+        data: Dict[str, List[Any]], fill_value: Optional[Any] = None
+    ) -> List[List[Any]]:
+        """
+        Transposes a matrix, filling in missing values with a specified fill value if needed.
+
+        Args:
+            data (Dict[str, List[Any]]): The matrix to be transposed.
+            fill_value (Optional[Any]): The value to fill in missing values with.
+
+        Returns:
+            The transposed matrix.
+        """  # noqa: E501
+        max_length = max(len(sublist) for sublist in data.values())
+
+        filled_data = {
+            key: value + [fill_value] * (max_length - len(value))
+            for key, value in data.items()
+        }
+
+        data_transposed = [list(row) for row in zip(*filled_data.values())]
+
+        return data_transposed
+
+    def _create_dataframe(self) -> pd.DataFrame:
+        """
+        Creates a DataFrame from the transposed data.
+
+        Returns:
+            The DataFrame representation of the data.
+        """
+        data_transposed = self._transpose_matrix(self.data)
+
+        df = pd.DataFrame(data_transposed, columns=self.field_options)
+
+        return df
+
+    def save_to_csv(self) -> None:
+        """
+        Saves the DataFrame to a CSV file.
+        """
+        csv_file_path = f"{self.filename}.csv"
+        self.dataframe.to_csv(csv_file_path, index=False)
+
+        print(f"Saved to {csv_file_path}")
+
+    def save_to_json(self) -> None:
+        """
+        Saves the DataFrame to a JSON file.
+        """
+        json_path = f"{self.filename}.json"
+        self.dataframe.to_json(json_path, orient="records", lines=False)
+
+        print(f"Saved to {json_path}")
+
+    def save_to_html(self) -> None:
+        """
+        Saves the DataFrame to an HTML file.
+        """
+        json_path = f"{self.filename}.json"
+
+        if not os.path.exists(json_path):
+            self.save_to_json()
+
+        html_file_path = f"{self.filename}.html"
+
+        html = HTMLTweetsVisualizer(self.username, json_path, html_file_path)
+
+        html_content = html.generate()
+        html.save(html_content)
+
+        print(f"Saved to {html_file_path}")

waybacktweets/api/parse.py

@@ -0,0 +1,295 @@
+"""
+Parses the returned data from the Wayback CDX Server API.
+"""
+
+import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from contextlib import nullcontext
+from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import unquote
+
+from rich import print as rprint
+from rich.progress import Progress
+
+from waybacktweets.config.config import config
+from waybacktweets.config.field_options import FIELD_OPTIONS
+from waybacktweets.exceptions.exceptions import (
+    ConnectionError,
+    GetResponseError,
+    HTTPError,
+)
+from waybacktweets.utils.utils import (
+    check_double_status,
+    check_pattern_tweet,
+    check_url_scheme,
+    clean_tweet_url,
+    delete_tweet_pathnames,
+    get_response,
+    is_tweet_url,
+    semicolon_parser,
+    timestamp_parser,
+)
+
+
+class TwitterEmbed:
+    """
+    This class is responsible for parsing tweets using the Twitter Publish service.
+
+    Args:
+        tweet_url (str): The URL of the tweet to be parsed.
+    """
+
+    def __init__(self, tweet_url: str):
+        self.tweet_url = tweet_url
+
+    def embed(self) -> Optional[Tuple[List[str], List[bool], List[str]]]:
+        """
+        Parses the archived tweets when they are still available.
+
+        This function goes through each archived tweet and checks if it is still available. If the tweet is available, it extracts the necessary information and adds it to the respective lists. The function returns a tuple of three lists:
+
+        - The first list contains the tweet texts.
+        - The second list contains boolean values indicating whether each tweet is still available.
+        - The third list contains the URLs of the tweets.
+
+        Returns:
+            A tuple of three lists containing the tweet texts, availability statuses, and URLs, respectively. If no tweets are available, returns None.
+        """  # noqa: E501
+        try:
+            url = f"https://publish.twitter.com/oembed?url={self.tweet_url}"
+            response = get_response(url=url)
+            if response:
+                json_response = response.json()
+                html = json_response["html"]
+                author_name = json_response["author_name"]
+
+                regex = re.compile(
+                    r'<blockquote class="twitter-tweet"(?: [^>]+)?><p[^>]*>(.*?)<\/p>.*?&mdash; (.*?)<\/a>',  # noqa
+                    re.DOTALL,
+                )
+                regex_author = re.compile(r"^(.*?)\s*\(")
+
+                matches_html = regex.findall(html)
+
+                tweet_content = []
+                user_info = []
+                is_RT = []
+
+                for match in matches_html:
+                    tweet_content_match = re.sub(
+                        r"<a[^>]*>|<\/a>", "", match[0].strip()
+                    ).replace("<br>", "\n")
+                    user_info_match = re.sub(
+                        r"<a[^>]*>|<\/a>", "", match[1].strip()
+                    ).replace(")", "), ")
+                    match_author = regex_author.search(user_info_match)
+                    author_tweet = match_author.group(1) if match_author else ""
+
+                    if tweet_content_match:
+                        tweet_content.append(tweet_content_match)
+                    if user_info_match:
+                        user_info.append(user_info_match)
+                        is_RT.append(author_name != author_tweet)
+
+                return tweet_content, is_RT, user_info
+        except ConnectionError:
+            if config.verbose:
+                rprint("[yellow]Error parsing the tweet, but the CDX data was saved.")
+        except HTTPError:
+            if config.verbose:
+                rprint(
+                    f"[yellow]{self.tweet_url} not available on the user's Twitter account, but the CDX data was saved."  # noqa: E501
+                )
+        except GetResponseError as e:
+            if config.verbose:
+                rprint(f"[red]An error occurred: {str(e)}")
+
+        return None
+
+
+class JsonParser:
+    """
+    This class is responsible for parsing tweets when the mimetype is application/json.
+
+    Note: This class is in an experimental phase.
+
+    Args:
+        archived_tweet_url (str): The URL of the archived tweet to be parsed.
+    """  # noqa: E501
+
+    def __init__(self, archived_tweet_url: str):
+        self.archived_tweet_url = archived_tweet_url
+
+    def parse(self) -> str:
+        """
+        Parses the archived tweets in JSON format.
+
+        Returns:
+            The parsed tweet text.
+        """
+        try:
+            response = get_response(url=self.archived_tweet_url)
+
+            if response:
+                json_data = response.json()
+
+                if "data" in json_data:
+                    return json_data["data"].get("text", json_data["data"])
+
+                if "retweeted_status" in json_data:
+                    return json_data["retweeted_status"].get(
+                        "text", json_data["retweeted_status"]
+                    )
+
+                return json_data.get("text", json_data)
+        except ConnectionError:
+            if config.verbose:
+                rprint(
+                    f"[yellow]Connection error with {self.archived_tweet_url}. Max retries exceeded. Error parsing the JSON, but the CDX data was saved."  # noqa: E501
+                )
+        except GetResponseError as e:
+            if config.verbose:
+                rprint(f"[red]An error occurred: {str(e)}")
+
+        return None
+
+
+class TweetsParser:
+    """
+    This class is responsible for the overall parsing of archived tweets.
+
+    Args:
+        archived_tweets_response (List[str]): The response from the archived tweets.
+        username (str): The username associated with the tweets.
+        field_options (List[str]): The fields to be included in the parsed data. For more details on each option, visit :ref:`field_options`.
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        archived_tweets_response: List[str],
+        username: str,
+        field_options: List[str],
+    ):
+        if not all(option in FIELD_OPTIONS for option in field_options):
+            raise ValueError("Some field options are not valid.")
+
+        self.archived_tweets_response = archived_tweets_response
+        self.username = username
+        self.field_options = field_options
+        self.parsed_tweets = {option: [] for option in self.field_options}
+
+    def _add_field(self, key: str, value: Any) -> None:
+        """
+        Appends a value to a list in the parsed data structure.
+
+        Args:
+            key (str): The key in the parsed data structure.
+            value (Any): The value to be appended.
+        """
+        if key in self.parsed_tweets:
+            self.parsed_tweets[key].append(value)
+
+    def _process_response(self, response: List[str]) -> None:
+        """
+        Processes the archived tweet's response and adds the relevant CDX data.
+
+        Args:
+            response (List[str]): The response from the archived tweet.
+        """
+        tweet_remove_char = unquote(response[2]).replace("’", "")
+        cleaned_tweet = check_pattern_tweet(tweet_remove_char).strip('"')
+
+        wayback_machine_url = (
+            f"https://web.archive.org/web/{response[1]}/{tweet_remove_char}"
+        )
+        original_tweet = delete_tweet_pathnames(
+            clean_tweet_url(cleaned_tweet, self.username)
+        )
+
+        double_status = check_double_status(wayback_machine_url, original_tweet)
+
+        if double_status:
+            original_tweet = delete_tweet_pathnames(
+                f"https://twitter.com{original_tweet}"
+            )
+        elif "://" not in original_tweet:
+            original_tweet = delete_tweet_pathnames(f"https://{original_tweet}")
+
+        parsed_wayback_machine_url = (
+            f"https://web.archive.org/web/{response[1]}/{original_tweet}"
+        )
+
+        encoded_archived_tweet = check_url_scheme(semicolon_parser(wayback_machine_url))
+        encoded_parsed_archived_tweet = check_url_scheme(
+            semicolon_parser(parsed_wayback_machine_url)
+        )
+        encoded_tweet = check_url_scheme(semicolon_parser(response[2]))
+        encoded_parsed_tweet = check_url_scheme(semicolon_parser(original_tweet))
+
+        available_tweet_text = None
+        available_tweet_is_RT = None
+        available_tweet_info = None
+
+        is_tweet = is_tweet_url(encoded_tweet)
+
+        if is_tweet:
+            embed_parser = TwitterEmbed(encoded_tweet)
+            content = embed_parser.embed()
+
+            if content:
+                available_tweet_text = semicolon_parser(content[0][0])
+                available_tweet_is_RT = content[1][0]
+                available_tweet_info = semicolon_parser(content[2][0])
+
+        self._add_field("available_tweet_text", available_tweet_text)
+        self._add_field("available_tweet_is_RT", available_tweet_is_RT)
+        self._add_field("available_tweet_info", available_tweet_info)
+
+        self._add_field("archived_urlkey", response[0])
+        self._add_field("archived_timestamp", response[1])
+        self._add_field("parsed_archived_timestamp", timestamp_parser(response[1]))
+        self._add_field("archived_tweet_url", encoded_archived_tweet)
+        self._add_field("parsed_archived_tweet_url", encoded_parsed_archived_tweet)
+        self._add_field("original_tweet_url", encoded_tweet)
+        self._add_field("parsed_tweet_url", encoded_parsed_tweet)
+        self._add_field("archived_mimetype", response[3])
+        self._add_field("archived_statuscode", response[4])
+        self._add_field("archived_digest", response[5])
+        self._add_field("archived_length", response[6])
+
+    def parse(self, print_progress=False) -> Dict[str, List[Any]]:
+        """
+        Parses the archived tweets CDX data and structures it.
+
+        Args:
+            print_progress (bool): A boolean indicating whether to print progress or not.
+
+        Returns:
+            The parsed tweets data.
+        """  # noqa: E501
+        with ThreadPoolExecutor(max_workers=10) as executor:
+
+            futures = {
+                executor.submit(self._process_response, response): response
+                for response in self.archived_tweets_response[1:]
+            }
+
+            progress_context = Progress() if print_progress else nullcontext()
+            with progress_context as progress:
+                task = None
+                if print_progress:
+                    task = progress.add_task(
+                        f"Parsing @{self.username}'s archived tweets\n",
+                        total=len(futures),
+                    )
+
+                for future in as_completed(futures):
+                    try:
+                        future.result()
+                    except Exception as e:
+                        rprint(f"[red]{e}")
+
+                    if print_progress:
+                        progress.update(task, advance=1)
+
+            return self.parsed_tweets

waybacktweets/api/request.py

@@ -0,0 +1,109 @@
+"""
+Requests data from the Wayback Machine API.
+"""
+
+from typing import Any, Dict, Optional
+
+from rich import print as rprint
+
+from waybacktweets.config.config import config
+from waybacktweets.exceptions.exceptions import (
+    ConnectionError,
+    EmptyResponseError,
+    GetResponseError,
+    HTTPError,
+    ReadTimeoutError,
+)
+from waybacktweets.utils.utils import get_response
+
+
+class WaybackTweets:
+    """
+    Class responsible for requesting data from the Wayback CDX Server API.
+
+    Args:
+        username (str): The username associated with the tweets.
+        collapse (str, optional): The field to collapse duplicate lines on.
+        timestamp_from (str, optional): The timestamp to start retrieving tweets from.
+        timestamp_to (str, optional): The timestamp to stop retrieving tweets at.
+        limit (int, optional): The maximum number of results to return.
+        offset (int, optional): The number of lines to skip in the results.
+        matchtype (str, optional): Results matching a certain prefix, a certain host or all subdomains.
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        username: str,
+        collapse: str = None,
+        timestamp_from: str = None,
+        timestamp_to: str = None,
+        limit: int = None,
+        offset: int = None,
+        matchtype: str = None,
+    ):
+        self.username = username
+        self.collapse = collapse
+        self.timestamp_from = timestamp_from
+        self.timestamp_to = timestamp_to
+        self.limit = limit
+        self.offset = offset
+        self.matchtype = matchtype
+
+    def get(self) -> Optional[Dict[str, Any]]:
+        """
+        Sends a GET request to the Internet Archive's CDX API to retrieve archived tweets.
+
+        Returns:
+            The response from the CDX API in JSON format, if successful. Otherwise, None.
+        """  # noqa: E501
+        url = "https://web.archive.org/cdx/search/cdx"
+
+        wildcard_pathname = "/*"
+        if self.matchtype:
+            wildcard_pathname = ""
+
+        params = {
+            "url": f"https://twitter.com/{self.username}/status{wildcard_pathname}",
+            "output": "json",
+        }
+
+        if self.collapse:
+            params["collapse"] = self.collapse
+
+        if self.timestamp_from:
+            params["from"] = self.timestamp_from
+
+        if self.timestamp_to:
+            params["to"] = self.timestamp_to
+
+        if self.limit:
+            params["limit"] = self.limit
+
+        if self.offset:
+            params["offset"] = self.offset
+
+        if self.matchtype:
+            params["matchType"] = self.matchtype
+
+        try:
+            response = get_response(url=url, params=params)
+            return response.json()
+        except ReadTimeoutError:
+            if config.verbose:
+                rprint("[red]Connection to web.archive.org timed out.")
+        except ConnectionError:
+            if config.verbose:
+                rprint(
+                    "[red]Failed to establish a new connection with web.archive.org. Max retries exceeded. Please wait a few minutes and try again."  # noqa: E501
+                )
+        except HTTPError as e:
+            if config.verbose:
+                rprint(f"[red]HTTP error occurred: {str(e)}")
+        except EmptyResponseError:
+            if config.verbose:
+                rprint("[red]No data was saved due to an empty response.")
+        except GetResponseError as e:
+            if config.verbose:
+                rprint(f"[red]An error occurred: {str(e)}")
+
+        return None

waybacktweets/api/visualize.py

@@ -0,0 +1,221 @@
+# flake8: noqa: E501
+"""
+Generates an HTML file to visualize the parsed data.
+"""
+
+import json
+import os
+from typing import Any, Dict, List, Union
+
+from waybacktweets.utils import timestamp_parser
+
+
+class HTMLTweetsVisualizer:
+    """
+    Class responsible for generating an HTML file to visualize the parsed data.
+
+    Args:
+        username (str): The username associated with the tweets.
+        json_path (Union[str, List[str]]): The path of the JSON file or the JSON data itself.
+        html_file_path (str, optional): The path where the HTML file will be saved.
+    """
+
+    def __init__(
+        self,
+        username: str,
+        json_path: Union[str, List[str]],
+        html_file_path: str = None,
+    ):
+        self.username = username
+        self.json_path = self._json_loader(json_path)
+        self.html_file_path = html_file_path
+
+    @staticmethod
+    def _json_loader(json_path: Union[str, List[str]]) -> List[Dict[str, Any]]:
+        """
+        Reads and loads JSON data from a specified file path or JSON string.
+
+        Args:
+            json_path (Union[str, List[str]]): The path of the JSON file or the JSON data itself.
+
+        Returns:
+            The content of the JSON file or data.
+        """
+        if os.path.isfile(json_path):
+            with open(json_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+
+        return json.loads(json_path)
+
+    def generate(self) -> str:
+        """
+        Generates an HTML string that represents the parsed data.
+
+        Returns:
+            The generated HTML string.
+        """
+        tweets_per_page = 24
+        total_pages = (len(self.json_path) + tweets_per_page - 1) // tweets_per_page
+
+        html = "<!DOCTYPE html>\n"
+        html += '<html lang="en">\n'
+        html += "<!-- This document was generated by Wayback Tweets. Visit: https://claromes.github.io/waybacktweets -->\n"
+
+        html += "<head>"
+        html += '<meta charset="UTF-8">\n'
+        html += (
+            '<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'
+        )
+        html += f"<title>@{self.username}'s archived tweets</title>\n"
+
+        # Adds styling
+        html += "<style>\n"
+        html += "body { font-family: monospace; background-color: whitesmoke; color: #1c1e21; margin: 0; padding: 20px; }\n"
+        html += ".container { display: flex; flex-wrap: wrap; gap: 20px; }\n"
+        html += ".tweet { flex: 0 1 calc(33.33% - 20px); background-color: #ffffff; border: 1px solid #e2e2e2; border-radius: 10px; padding: 15px; overflow-wrap: break-word; margin: auto; width: 600px; }\n"
+        html += ".tweet strong { font-weight: bold; }\n"
+        html += ".tweet a { color: #000000; text-decoration: none; }\n"
+        html += ".content { color: #000000; }\n"
+        html += ".source { font-size: 12px; text-align: center; }\n"
+        html += ".tweet a:hover { text-decoration: underline; }\n"
+        html += "h1, h3 { text-align: center; }\n"
+        html += "iframe { width: 600px; height: 600px; }\n"
+        html += "input { position: absolute; opacity: 0; z-index: -1; }\n"
+        html += ".accordion { margin: 10px; border-radius: 5px; overflow: hidden; box-shadow: 0 4px 4px -2px rgba(0, 0, 0, 0.4); }\n"
+        html += ".accordion-label { display: flex; justify-content: space-between; padding: 1em; font-weight: bold; cursor: pointer; background: #000000; color: #ffffff; }\n"
+        html += ".accordion-content { max-height: 0; padding: 0 1em; background: white; transition: all 0.35s; }\n"
+        html += (
+            "input:checked ~ .accordion-content { max-height: 100vh; padding: 1em; }\n"
+        )
+        html += ".pagination { text-align: center; margin-top: 20px; }\n"
+        html += ".pagination a { margin: 0 5px; text-decoration: none; color: #000000; padding: 1px 2px; border-radius: 5px; }\n"
+        html += ".pagination a:hover { background-color: #e2e2e2; }\n"
+        html += ".pagination a.selected { background-color: #e2e2e2; color: #000000; font-weight: bold; }\n"
+        html += "</style>\n"
+
+        html += "</head>\n<body>\n"
+
+        html += f"<h1>@{self.username}'s archived tweets</h1>\n"
+
+        html += (
+            '<p id="loading_first_page">Building pagination with JavaScript...</p>\n'
+        )
+
+        for page in range(1, total_pages + 1):
+            html += (
+                f'<div id="page_{page}" style="display:none;">\n'  # Starts a new page
+            )
+            html += '<div class="container">\n'
+
+            start_index = (page - 1) * tweets_per_page
+            end_index = min(start_index + tweets_per_page, len(self.json_path))
+
+            for index in range(start_index, end_index):
+                tweet = self.json_path[index]
+                html += '<div class="tweet">\n'
+
+                if not tweet["available_tweet_text"]:
+                    iframe_src = {
+                        "Archived Tweet": tweet["archived_tweet_url"],
+                        "Parsed Archived Tweet": tweet["parsed_archived_tweet_url"],
+                        "Original Tweet": tweet["original_tweet_url"],
+                        "Parsed Tweet": tweet["parsed_tweet_url"],
+                    }
+
+                    for key, value in iframe_src.items():
+                        key_cleaned = key.replace(" ", "_")
+
+                        html += '<div class="accordion">\n'
+                        html += f'<input type="checkbox" id="tab_{index}_{key_cleaned}" />\n'
+                        html += f'<label class="accordion-label" for="tab_{index}_{key_cleaned}">Click to load the iframe from {key}</label>\n'
+                        html += '<div class="accordion-content">\n'
+
+                        html += f'<div id="loading_{index}_{key_cleaned}" class="loading">Loading...</div>\n'
+                        html += f'<iframe id="iframe_{index}_{key_cleaned}" frameborder="0" scrolling="auto" loading="lazy" style="display: none;" onload="document.getElementById(\'loading_{index}_{key_cleaned}\').style.display=\'none\'; this.style.display=\'block\';"></iframe>\n'
+                        html += "</div>\n"
+                        html += "</div>\n"
+
+                        html += """
+                        <script>
+                        // Loads the src attribute of the iframe tag
+                        document.getElementById('tab_{index}_{key_cleaned}').addEventListener('change', function() {{
+                            if (this.checked) {{
+                                document.getElementById('loading_{index}_{key_cleaned}').style.display = 'block';
+                                document.getElementById('iframe_{index}_{key_cleaned}').src = '{url}';
+                            }}
+                        }});
+                        </script>
+                        """.format(
+                            index=index, url=value, key_cleaned=key_cleaned
+                        )
+
+                if tweet["available_tweet_text"]:
+                    html += "<br>\n"
+                    html += f'<p><strong class="content">Available Tweet Content:</strong> {tweet["available_tweet_text"]}</p>\n'
+                    html += f'<p><strong class="content">Available Tweet Is Retweet:</strong> {tweet["available_tweet_is_RT"]}</p>\n'
+                    html += f'<p><strong class="content">Available Tweet Username:</strong> {tweet["available_tweet_info"]}</p>\n'
+
+                html += "<br>\n"
+                html += f'<p><strong>Archived Tweet:</strong> <a href="{tweet["archived_tweet_url"]}" target="_blank">{tweet["archived_tweet_url"]}</a></p>\n'
+                html += f'<p><strong>Parsed Archived Tweet:</strong> <a href="{tweet["parsed_archived_tweet_url"]}" target="_blank">{tweet["parsed_archived_tweet_url"]}</a></p>\n'
+                html += f'<p><strong>Original Tweet:</strong> <a href="{tweet["original_tweet_url"]}" target="_blank">{tweet["original_tweet_url"]}</a></p>\n'
+                html += f'<p><strong>Parsed Tweet:</strong> <a href="{tweet["parsed_tweet_url"]}" target="_blank">{tweet["parsed_tweet_url"]}</a></p>\n'
+                html += f'<p><strong>Archived URL Key:</strong> {tweet["archived_urlkey"]}</p>\n'
+                html += f'<p><strong>Archived Timestamp:</strong> {timestamp_parser(tweet["archived_timestamp"])} ({tweet["archived_timestamp"]})</p>\n'
+                html += f'<p><strong>Archived mimetype:</strong> {tweet["archived_mimetype"]}</p>\n'
+                html += f'<p><strong>Archived Statuscode:</strong> {tweet["archived_statuscode"]}</p>\n'
+                html += (
+                    f'<p><strong>Archived Digest:</strong> {tweet["archived_digest"]}\n'
+                )
+                html += f'<p><strong>Archived Length:</strong> {tweet["archived_length"]}</p>\n'
+                html += "</div>\n"
+
+            html += "</div>\n</div>\n"  # Closes the page div and the container
+
+        html += "<br>\n"
+
+        # Adds navigation for the pages
+        html += '<div class="pagination">\n'
+        for page in range(1, total_pages + 1):
+            html += f'<a href="#" id="page_link_{page}" onclick="showPage({page})">{page}</a>\n'
+        html += "</div>\n"
+
+        html += '<br><p class="source">generated by <a href="https://claromes.github.io/waybacktweets/" target="_blank">Wayback Tweets↗</a></p>\n'
+
+        html += """
+        <script>
+        // Function to show the selected page and hide the others
+        function showPage(page) {{
+            for (let i = 1; i <= {total_pages}; i++) {{
+                document.getElementById('page_' + i).style.display = 'none';
+                document.getElementById('page_link_' + i).classList.remove('selected');
+            }}
+
+            document.getElementById('page_' + page).style.display = 'block';
+            document.getElementById('page_link_' + page).classList.add('selected');
+        }}
+
+        // Initializes the page to show only the first page
+        document.addEventListener('DOMContentLoaded', (event) => {{
+            showPage(1); // Shows only the first page on load
+            document.getElementById('loading_first_page').style.display = 'none';
+        }});
+        </script>
+        """.format(
+            total_pages=total_pages
+        )
+
+        html += "</body>\n"
+        html += "</html>"
+
+        return html
+
+    def save(self, html_content: str) -> None:
+        """
+        Saves the generated HTML string to a file.
+
+        Args:
+            html_content (str): The HTML string to be saved.
+        """
+        with open(self.html_file_path, "w", encoding="utf-8") as f:
+            f.write(html_content)

waybacktweets/config/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+
+from waybacktweets.config.config import config
+from waybacktweets.config.field_options import FIELD_OPTIONS

waybacktweets/config/config.py

@@ -0,0 +1,28 @@
+"""
+Configuration module.
+
+Manages global configuration settings throughout the application.
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class _Config:
+    """
+    A class used to represent the configuration settings.
+
+    Attributes:
+        verbose (bool): Determines if verbose logging should be enabled.
+    """
+
+    verbose: bool = True
+
+
+config = _Config()
+"""
+Global configuration instance.
+
+Attributes:
+    verbose (bool): Determines if verbose logging should be enabled.
+"""

waybacktweets/config/field_options.py

@@ -0,0 +1,20 @@
+"""
+List of valid field options that can be used for parsing tweets.
+"""
+
+FIELD_OPTIONS = [
+    "archived_urlkey",
+    "archived_timestamp",
+    "parsed_archived_timestamp",
+    "archived_tweet_url",
+    "parsed_archived_tweet_url",
+    "original_tweet_url",
+    "parsed_tweet_url",
+    "available_tweet_text",
+    "available_tweet_is_RT",
+    "available_tweet_info",
+    "archived_mimetype",
+    "archived_statuscode",
+    "archived_digest",
+    "archived_length",
+]

waybacktweets/exceptions/__init__.py

@@ -0,0 +1,9 @@
+# flake8: noqa: F401
+
+from waybacktweets.exceptions.exceptions import (
+    ConnectionError,
+    EmptyResponseError,
+    GetResponseError,
+    HTTPError,
+    ReadTimeoutError,
+)

waybacktweets/exceptions/exceptions.py

@@ -0,0 +1,33 @@
+"""
+Wayback Tweets Exceptions
+"""
+
+
+class GetResponseError(Exception):
+    """
+    Base class for exceptions in get_response.
+    """
+
+
+class ReadTimeoutError(GetResponseError):
+    """
+    Exception raised for read timeout errors.
+    """
+
+
+class ConnectionError(GetResponseError):
+    """
+    Exception raised for connection errors.
+    """
+
+
+class HTTPError(GetResponseError):
+    """
+    Exception raised for HTTP errors.
+    """
+
+
+class EmptyResponseError(GetResponseError):
+    """
+    Exception raised for empty responses.
+    """

waybacktweets/utils/__init__.py

@@ -0,0 +1,14 @@
+# flake8: noqa: F401
+
+from waybacktweets.utils.utils import (
+    check_double_status,
+    check_pattern_tweet,
+    check_url_scheme,
+    clean_tweet_url,
+    clean_wayback_machine_url,
+    delete_tweet_pathnames,
+    get_response,
+    is_tweet_url,
+    semicolon_parser,
+    timestamp_parser,
+)

waybacktweets/utils/utils.py

@@ -0,0 +1,279 @@
+"""
+Utility functions for handling HTTP requests and manipulating URLs.
+"""
+
+import html
+import re
+from datetime import datetime
+from typing import Optional, Tuple
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from waybacktweets.exceptions.exceptions import (
+    ConnectionError,
+    EmptyResponseError,
+    GetResponseError,
+    HTTPError,
+    ReadTimeoutError,
+)
+
+
+def get_response(
+    url: str, params: Optional[dict] = None
+) -> Tuple[Optional[requests.Response], Optional[str], Optional[str]]:
+    """
+    Sends a GET request to the specified URL and returns the response.
+
+    Args:
+        url (str): The URL to send the GET request to.
+        params (dict, optional): The parameters to include in the GET request.
+
+    Returns:
+        The response from the server.
+
+    Raises:
+        ReadTimeoutError: If a read timeout occurs.
+        ConnectionError: If a connection error occurs.
+        HTTPError: If an HTTP error occurs.
+        EmptyResponseError: If the response is empty.
+    """
+    session = requests.Session()
+    retry = Retry(connect=3, backoff_factor=0.3)
+    adapter = HTTPAdapter(max_retries=retry)
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"  # noqa: E501
+    }
+
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+
+    try:
+        response = session.get(url, params=params, headers=headers)
+        response.raise_for_status()
+
+        if not response or response.json() == []:
+            raise EmptyResponseError("No data was saved due to an empty response.")
+        return response
+    except requests.exceptions.ReadTimeout:
+        raise ReadTimeoutError
+    except requests.exceptions.ConnectionError:
+        raise ConnectionError
+    except requests.exceptions.HTTPError:
+        raise HTTPError
+    except requests.exceptions.RequestException:
+        raise GetResponseError
+
+
+def clean_tweet_url(tweet_url: str, username: str) -> str:
+    """
+    Cleans a tweet URL by ensuring it is associated with the correct username.
+
+    Args:
+        tweet_url (str): The tweet URL to clean.
+        username (str): The username to associate with the tweet URL.
+
+    Returns:
+        The cleaned tweet URL.
+    """
+    tweet_lower = tweet_url.lower()
+
+    pattern = re.compile(r"/status/(\d+)")
+    match_lower_case = pattern.search(tweet_lower)
+    match_original_case = pattern.search(tweet_url)
+
+    if match_lower_case and username in tweet_lower:
+        return f"https://twitter.com/{username}/status/{match_original_case.group(1)}"
+    else:
+        return tweet_url
+
+
+def clean_wayback_machine_url(
+    wayback_machine_url: str, archived_timestamp: str, username: str
+) -> str:
+    """
+    Cleans a Wayback Machine URL by ensuring it is associated with the correct username and timestamp.
+
+    Args:
+        wayback_machine_url (str): The Wayback Machine URL to clean.
+        archived_timestamp (str): The timestamp to associate with the Wayback Machine URL.
+        username (str): The username to associate with the Wayback Machine URL.
+
+    Returns:
+        The cleaned Wayback Machine URL.
+    """  # noqa: E501
+    wayback_machine_url = wayback_machine_url.lower()
+
+    pattern = re.compile(r"/status/(\d+)")
+    match = pattern.search(wayback_machine_url)
+
+    if match and username in wayback_machine_url:
+        return f"https://web.archive.org/web/{archived_timestamp}/https://twitter.com/{username}/status/{match.group(1)}"  # noqa: E501
+    else:
+        return wayback_machine_url
+
+
+def check_pattern_tweet(tweet_url: str) -> str:
+    """
+    Extracts the URL from a tweet URL with patterns such as:
+
+    - Reply: /status//
+    - Link:  /status///
+    - Twimg: /status/https://pbs
+
+    Args:
+        tweet_url (str): The tweet URL to extract the URL from.
+
+    Returns:
+        Only the extracted URL from a tweet.
+    """
+    pattern = r'/status/((?:"(.*?)"|"(.*?)(?=&|$)|&quot%3B(.*?)(?=&|$)))'
+    match = re.search(pattern, tweet_url)
+
+    if match:
+        if match.group(2):
+            parsed_tweet_url = match.group(2)
+        elif match.group(3):
+            parsed_tweet_url = match.group(3)
+        elif match.group(4):
+            parsed_tweet_url = match.group(4)
+        else:
+            parsed_tweet_url = ""
+
+        parsed_tweet_url = html.unescape(parsed_tweet_url)
+
+        return parsed_tweet_url
+
+    return tweet_url
+
+
+def delete_tweet_pathnames(tweet_url: str) -> str:
+    """
+    Removes any pathnames from a tweet URL.
+
+    Args:
+        tweet_url (str): The tweet URL to remove pathnames from.
+
+    Returns:
+        The tweet URL without any pathnames.
+    """
+    pattern_username = re.compile(r"https://twitter\.com/([^/]+)/status/\d+")
+    match_username = pattern_username.match(tweet_url)
+
+    pattern_id = r"https://twitter.com/\w+/status/(\d+)"
+    match_id = re.search(pattern_id, tweet_url)
+
+    if match_id and match_username:
+        tweet_id = match_id.group(1)
+        username = match_username.group(1)
+        return f"https://twitter.com/{username}/status/{tweet_id}"
+    else:
+        return tweet_url
+
+
+def check_double_status(wayback_machine_url: str, original_tweet_url: str) -> bool:
+    """
+    Checks if a Wayback Machine URL contains two occurrences of "/status/" and if the original tweet does not contain "twitter.com".
+
+    Args:
+        wayback_machine_url (str): The Wayback Machine URL to check.
+        original_tweet_url (str): The original tweet URL to check.
+
+    Returns:
+        True if the conditions are met, False otherwise.
+    """  # noqa: E501
+    if (
+        wayback_machine_url.count("/status/") == 2
+        and "twitter.com" not in original_tweet_url
+    ):
+        return True
+
+    return False
+
+
+def semicolon_parser(string: str) -> str:
+    """
+    Replaces semicolons in a string with %3B.
+
+    Args:
+        string (str): The string to replace semicolons in.
+
+    Returns:
+        The string with semicolons replaced by %3B.
+    """
+    return "".join("%3B" if c == ";" else c for c in string)
+
+
+def is_tweet_url(twitter_url: str) -> bool:
+    """
+    Checks if the provided URL is a Twitter status URL.
+
+    This function checks if the provided URL contains "/status/" exactly once, which is a common pattern in Twitter status URLs.
+
+    Args:
+        twitter_url (str): The URL to check.
+
+    Returns:
+        True if the URL is a Twitter status URL, False otherwise.
+    """  # noqa: E501
+    if twitter_url.count("/status/") == 1:
+        return True
+
+    return False
+
+
+def timestamp_parser(timestamp):
+    """
+    Parses a timestamp into a formatted string.
+
+    Args:
+        timestamp (str): The timestamp string to parse.
+
+    Returns:
+        The parsed timestamp in the format "%Y/%m/%d %H:%M:%S", or None if the
+        timestamp could not be parsed.
+    """
+    formats = [
+        "%Y",
+        "%Y%m",
+        "%Y%m%d",
+        "%Y%m%d%H",
+        "%Y%m%d%H%M",
+        "%Y%m%d%H%M%S",
+    ]
+
+    for fmt in formats:
+        try:
+            parsed_time = datetime.strptime(timestamp, fmt)
+
+            formatted_time = parsed_time.strftime("%Y/%m/%d %H:%M:%S")
+            return formatted_time
+        except ValueError:
+            continue
+
+    return None
+
+
+def check_url_scheme(url):
+    """
+    Corrects the URL scheme if it contains more than two slashes following the scheme.
+
+    This function uses a regular expression to find 'http:' or 'https:' followed by two or more slashes.
+    It then replaces this with the scheme followed by exactly two slashes.
+
+    Args:
+        url (str): The URL to be corrected.
+
+    Returns:
+        The corrected URL.
+    """  # noqa: E501
+    pattern = r"(http:|https:)(/{2,})"
+
+    def replace_function(match):
+        scheme = match.group(1)
+        return f"{scheme}//"
+
+    parsed_url = re.sub(pattern, replace_function, url)
+
+    return parsed_url