clean & refactor components + add doc

app.py

@@ -1,451 +1,102 @@
 #!/usr/bin/env python3
 # -*- coding:utf-8 -*-
-import requests.exceptions
-import zipfile
 
 import streamlit as st
-from streamlit.components.v1 import html
-from n4a_analytics_lib.analytics import (GlobalStatistics, IaaStatistics)
-from n4a_analytics_lib.constants import (DESCRIPTION)
 
-
-# Set application
-st.set_page_config(layout="wide")
-# sidebar: meta, inputs etc.
-sidebar = st.sidebar
-# cols: display results
-col1, col2 = st.columns(2)
-
-# description
-sidebar.markdown(DESCRIPTION)
-
-
-
-
-# to st components
-#def clear_cache():
-#    st.session_state = {}
-
-def check_login(username, password):
-    if (len(username) == 0) or (len(password) == 0):
-        return False
-    return True
-
-def logout():
-    pass
-
-
-
-
-
-
-# Level to analyze
-option = sidebar.selectbox('Which statistics level?', ('Inter-Annotator Agreement results',
-                                                       'Global project statistics'))
-
-# IAA results view
-if option == "Inter-Annotator Agreement results":
-    annotations = sidebar.file_uploader("Upload IAA annotations (.zip format only): ")
-    baseline_text = sidebar.file_uploader("Upload baseline text (.txt format only): ")
-
-
-    if baseline_text is not None and annotations is not None:
-        project_analyzed = IaaStatistics(zip_project=annotations, baseline_text=baseline_text.getvalue())
-        baseline_analyzer = project_analyzed.analyze_text()
-
-        col2.markdown(f"""
-        ### BASELINE TEXT: {baseline_text.name}
-
-         - sentences:  {baseline_analyzer[0]}
-         - words: {baseline_analyzer[1]}
-         - characters: {baseline_analyzer[2]}
-        """)
-
-
-
-
-        #print(project_analyzed.annotations_per_coders)
-
-        commune_mentions = [l for i,j in project_analyzed.mentions_per_coder.items() for l in j]
-        commune_mentions = list(dict.fromkeys(commune_mentions))
-        #print(commune_mentions)
-        #print(project_analyzed.annotations)
-        #print(project_analyzed.labels_per_coder)
-        import pandas as pd
-        from collections import defaultdict, Counter
-        from itertools import combinations
-        import seaborn as sn
-        import matplotlib as plt
-        import matplotlib.pyplot as pylt
-
-        dicts_coders = []
-        for coder, annotations in project_analyzed.annotations_per_coders.items():
-            nombre_annotations = []
-            # print(f'* {coder}')
-            for annotation, label in annotations.items():
-                nombre_annotations.append(label)
-            # print(f"Nombre total d'annotations : {len(nombre_annotations)}")
-            dict_coder = dict(Counter(nombre_annotations))
-            dicts_coders.append(dict_coder)
-            # print(f'==========================')
-
-        labels = [label for label in dicts_coders[0]]
-
-        from n4a_analytics_lib.metrics_utils import interpret_kappa, fleiss_kappa_function, cohen_kappa_function
-        df = pd.DataFrame(project_analyzed.annotations_per_coders, index=commune_mentions)
-
-        for ann in project_analyzed.annotators:
-            df[ann] = 'None'
-            for mention, value in project_analyzed.annotations_per_coders[ann].items():
-                df.loc[mention, ann] = value
-
-        total_annotations = len(df)
-
-        # print(f'* Total des annotations : {total_annotations}')
-
-        df_n = df.apply(pd.Series.value_counts, 1).fillna(0).astype(int)
-        matrix = df_n.values
-
-        pairs = list(combinations(project_analyzed.annotations_per_coders, 2))
-
-        # Display in app
-        #cont_kappa = st.container()
-        st.title("Inter-Annotator Agreement (IAA) results")
-        #tab1, tab2, tab3, tab4, tab5 = st.tabs(
-        #    ["📈 IAA metrics", "🗃 IAA Metrics Legend", "✔️ Agree annotations", "❌ Disagree annotations",
-        #     "🏷️ Global Labels Statistics"])
-        st.markdown("## 📈 IAA metrics")
-        col1_kappa, col2_kappa = st.columns(2)
-        col1_kappa.subheader("Fleiss Kappa (global score for group):")
-
-
-        col1_kappa.markdown(interpret_kappa(round(fleiss_kappa_function(matrix), 2)), unsafe_allow_html=True)
-        col1_kappa.subheader("Cohen Kappa Annotators Matrix (score between annotators):")
-        # tab1.dataframe(df)
-        data = []
-        for coder_1, coder_2 in pairs:
-            cohen_function = cohen_kappa_function(project_analyzed.labels_per_coder[coder_1], project_analyzed.labels_per_coder[coder_2])
-            data.append(((coder_1, coder_2), cohen_function))
-            col1_kappa.markdown(f"* {coder_1} <> {coder_2} : {interpret_kappa(cohen_function)}", unsafe_allow_html=True)
-            # print(f"* {coder_1} <> {coder_2} : {cohen_function}")
-
-        intermediary = defaultdict(Counter)
-        for (src, tgt), count in data:
-            intermediary[src][tgt] = count
-
-        letters = sorted({key for inner in intermediary.values() for key in inner} | set(intermediary.keys()))
-
-        confusion_matrix = [[intermediary[src][tgt] for tgt in letters] for src in letters]
-        import numpy as np
-
-        df_cm = pd.DataFrame(confusion_matrix, letters, letters)
-        mask = df_cm.values == 0
-        sn.set(font_scale=0.7)  # for label size
-        colors = ["#e74c3c", "#f39c12", "#f4d03f", "#5dade2", "#58d68d", "#28b463"]
-        width = st.slider("matrix width", 1, 10, 14)
-        height = st.slider("matrix height", 1, 10, 4)
-        fig, ax = pylt.subplots(figsize=(width, height))
-        sn.heatmap(df_cm, cmap=colors, annot=True, mask=mask, annot_kws={"size": 7}, vmin=0, vmax=1, ax=ax)  # font size
-        # plt.show()
-        st.pyplot(ax.figure)
-        col2_kappa.markdown("""
-        <div>
-        <div id="legend" style="right: 70em;">
-        <h3>🗃 IAA Metrics Legend</h3>
-        <table>
-        <thead>
-        <tr>
-        <th
-        colspan="2"> Kappa
-        interpretation
-        legend </th>
-                   </tr>
-                       </thead>
-                           <tbody>
-                           <tr>
-                           <td> Kappa
-        score(k) </td>
-                     <td>Agreement</td>
-                                          </tr>
-                                              <tr
-        style = "background-color: #e74c3c;">
-                <td> k < 0 </td>
-                                 <td> Less
-        chance
-        agreement </td>
-                      </tr>
-                          <tr
-        style = "background-color: #f39c12;">
-                <td> 0.01 < k < 0.20 </td>
-                                           <td> Slight
-        agreement </td>
-                      </tr>
-                          <tr
-        style = "background-color: #f4d03f;">
-                <td> 0.21 < k < 0.40 </td>
-                                           <td> Fair
-        agreement </td>
-                      </tr>
-                          <tr
-        style = "background-color:  #5dade2;">
-                <td> 0.41 < k < 0.60 </td>
-                                           <td> Moderate
-        agreement </td>
-                      </tr>
-                          <tr
-        style = "background-color:  #58d68d;">
-                <td> 0.61 < k < 0.80 </td>
-                                           <td> Substantial
-        agreement </td>
-                      </tr>
-                          <tr
-        style = "background-color:  #28b463;">
-                <td> 0.81 < k < 0.99 </td>
-                                           <td> Almost
-        perfect
-        agreement </td>
-                      </tr>
-                          </tbody>
-                              </table></div></div>"""
-
-        , unsafe_allow_html = True)
-
-
-        ## commune
-        @st.cache
-        def convert_df(df_ex):
-            return df_ex.to_csv(encoding="utf-8").encode('utf-8')
-
-
-        ## Agree part
-
-        columns_to_compare = project_analyzed.annotators
-
-
-        def check_all_equal(iterator):
-            return len(set(iterator)) <= 1
-
-
-        df_agree = df[df[columns_to_compare].apply(lambda row: check_all_equal(row), axis=1)]
-        total_unanime = len(df_agree)
-
-        csv_agree = convert_df(df_agree)
-
-        st.subheader("✔️ Agree annotations")
-        st.markdown(f"{total_unanime} / {len(df)} annotations ({round((total_unanime / len(df)) * 100, 2)} %)")
-        st.download_button(
-            "Press to Download CSV",
-            csv_agree,
-            "csv_annotators_agree.csv",
-            "text/csv",
-            key='download-csv-1'
+from n4a_analytics_lib.constants import DESCRIPTION
+
+from n4a_analytics_lib.st_components import (check_login,
+                                             init_session_statistics,
+                                             init_session_iaa,
+                                             display_data)
+
+
+def n4a_analytics_dashboard() -> None:
+    """Main function to manage dashboard app frontend
+    -------------------------------------------------
+    * General architecture:
+    *
+    *                            metrics_utils.py (collection of statistics calculation)
+    *                                                     ↓
+    *  project.py (features extraction from XMI) → analytics.py
+    *                             ↑               (project analyzer: computation/visualisation)
+    *                             ↑                       ↓
+    *      st_components.py (manage data input/output and pipelines with streamlit snippets)
+    *                                        ↑ ↓
+    *                               app.py (manage frontend)
+    *
+    ---------------------------------------------------
+    """
+    # Set window application
+    st.set_page_config(layout="wide")
+
+    # Sidebar: metadata, inputs etc.
+    sidebar = st.sidebar
+    # Cols: display results
+    col1, col2 = st.columns(2)
+
+    # Set general description
+    sidebar.markdown(DESCRIPTION)
+
+    # Level to analyze
+    option = sidebar.selectbox('Which statistics level?', ('Inter-Annotator Agreement results',
+                                                           'Global project statistics'))
+
+    # IAA results view
+    if option == "Inter-Annotator Agreement results":
+        annotations = sidebar.file_uploader(
+            "Upload IAA annotations (.zip format only): ",
+            type='zip'
         )
-        st.dataframe(df_agree)
-
-
-        ## Disagree part
-
-        def check_all_not_equal(iterator):
-            return len(set(iterator)) > 1
-
-
-        df_disagree = df[df[columns_to_compare].apply(lambda row: check_all_not_equal(row), axis=1)]
-        total_desaccord = len(df_disagree)
-        csv_disagree = convert_df(df_disagree)
-        st.subheader("❌ Disagree annotations")
-        st.markdown(
-            f"{total_desaccord} / {len(df)} annotations ({round((total_desaccord / len(df)) * 100, 2)} %)")
-        st.download_button(
-            "Press to Download CSV",
-            csv_disagree,
-            "csv_annotators_disagree.csv",
-            "text/csv",
-            key='download-csv-2'
+        baseline_text = sidebar.file_uploader(
+            "Upload baseline text (.txt format only): ",
+            type='txt'
         )
-        st.dataframe(df_disagree)
-
-
-        ## alignement chart labels
-        def count_total_annotations_label(dataframe, labels):
-            pairs = []
-            for label in labels:
-                total = dataframe.astype(object).eq(label).any(1).sum()
-                pairs.append((label, total))
-            return pairs
-
-
-        totals_annotations_per_labels = count_total_annotations_label(df, labels)
 
-
-        # Récupérer le nombre de mention portant la même classe selon les annotateurs
-
-        def total_agree_disagree_per_label(dataframe, pairs_totals_labels):
-            new_pairs = []
-            for t in pairs_totals_labels:
-                # t[0] : label
-                # t[1] : total_rows_with_label
-                agree_res = df[df.nunique(1).eq(1)].eq(t[0]).any(1).sum()
-                disagree_res = t[1] - agree_res
-                agree_percent = (agree_res / t[1]) * 100
-                disagree_percent = (disagree_res / t[1]) * 100
-                new_pairs.append((t[0], t[1], agree_percent, disagree_percent))
-            return new_pairs
-
-        to_pie = total_agree_disagree_per_label(df, totals_annotations_per_labels)
-
-
-        def plot_pies(tasks_to_pie):
-         my_labels = 'agree', 'disagree'
-         my_colors = ['#47DBCD', '#F5B14C']
-         my_explode = (0, 0.1)
-         counter = 0
-         fig, axes = pylt.subplots(1, len(tasks_to_pie), figsize=(20, 3))
-         for t in tasks_to_pie:
-             tasks = [t[2], t[3]]
-             axes[counter].pie(tasks, autopct='%1.1f%%', startangle=15, shadow=True, colors=my_colors,
-                               explode=my_explode)
-             axes[counter].set_title(t[0])
-             axes[counter].axis('equal')
-             counter += 1
-         fig.set_facecolor("white")
-         fig.legend(labels=my_labels, loc="center right", borderaxespad=0.1, title="Labels alignement")
-         # plt.savefig(f'./out/pie_alignement_labels_{filename_no_extension}.png', dpi=400)
-         return fig
-
-        f = plot_pies(to_pie)
-        st.subheader("🏷️ Global Labels Statistics")
-        st.pyplot(f.figure)
-
-# global project results view
-# st_session = {"gs_local":True, "gs_remote":False, "gs_obj":<object>}
-
-def display_data():
-    col1.metric("Total curated annotations",
-                f"{st.session_state['gs_obj'].total_annotations_project} Named entities")
-    col1.dataframe(st.session_state['gs_obj'].df_i)
-    selected_data = col1.selectbox('Select specific data to display bar plot:',
-                                   st.session_state['gs_obj'].documents, key="selector_data")
-    col2.pyplot(st.session_state['gs_obj'].create_plot(selected_data))
-
-def init_session_statistics(remote: bool, local: bool, data: tuple) -> None:
-    # clear session
-    st.session_state = {}
-
-    # create a session variable
-    st.session_state["gs_local"] = local
-    st.session_state["gs_remote"] = remote
-
-    # create a new object:
-    # if remote fetch data from API Host first
-    if remote and not(local):
-        st.success('Fetch curated documents from host INCEpTION API in progress...')
-        fetch_curated_data_from_remote(
-                username=data[0],
-                password=data[1]
-        )
-
-    if local and not(remote):
-        st.session_state["gs_obj"] = GlobalStatistics(zip_project=data, remote=False)
-
-
-
-
-
-from pycaprio import Pycaprio, mappings
-from zipfile import ZipFile
-import io
-import requests
-
-def fetch_curated_data_from_remote(username: str,
-                                   password: str,
-                                   endpoint: str = "https://inception.dhlab.epfl.ch/prod",
-                                   project_title: str = "ner4archives-template"):
-    # open a client
-    try:
-        client = Pycaprio(inception_host=endpoint, authentication=(str(username), str(password)))
-    except requests.exceptions.JSONDecodeError:
-        # username / password incorrect
-        st.error('Username or Password is incorrect please retry.')
-
-    # get project object
-    project_name = [p for p in client.api.projects() if p.project_name == project_title]
-
-    # get all documents from project
-    documents = client.api.documents(project_name[0].project_id)
-
-    curations = []
-    zipfiles = []
-    count = 0
-    flag = "a"
-    # iterate over all documents and retrieve only curated into ZIP container
-    for document in documents:
-        if count > 0:
-            flag = "r"
-        if document.document_state == mappings.DocumentState.CURATION_COMPLETE:
-            curated_content = client.api.curation(project_name[0].project_id, document,
-                                                  curation_format=mappings.InceptionFormat.UIMA_CAS_XMI_XML_1_1)
-            curations.append(curated_content)
-            for curation in curations:
-                z = ZipFile(io.BytesIO(curation), mode=flag)
-                zipfiles.append(z)
-
-        count += 1
-
-    # Merge all zip in one
-    with zipfiles[0] as z1:
-        for fname in zipfiles[1:]:
-            zf = fname
-            # print(zf.namelist())
-            for n in zf.namelist():
-                if n not in z1.namelist():
-                    z1.writestr(n, zf.open(n).read())
-
-        # Create a new object
-        st.session_state["gs_obj"] = GlobalStatistics(zip_project=z1, remote=True)
-
-
-
-
-if option == "Global project statistics":
-    # User input controllers
-    mode = sidebar.radio("Choose mode to retrieve curated data: ", (
-        "Local directory", "INCEpTION API Host remote"
-    ))
-    data = None
-    if mode == "Local directory":
-        project = sidebar.file_uploader("Folder that contains curated annotations in XMI 1.1 (.zip format only): ",
-                                        type="zip")
-        data = project
-    if mode == "INCEpTION API Host remote":
-        username = sidebar.text_input("Username: ")
-        password = sidebar.text_input("Password: ", type='password')
-        data = (username, password)
-
-    # Validate inputs
-    btn_process = sidebar.button('Process', key='process')
-
-    # Access data with local ressources
-    if btn_process and mode == "Local directory":
-        if data is not None:
-            # create a new session
-            init_session_statistics(remote=False, local=True, data=data)
-
-    # Access data with remote ressources
-    if btn_process and mode == "INCEpTION API Host remote":
-        if data is not None:
-            if check_login(username=data[0], password=data[1]):
+        if baseline_text is not None and annotations is not None:
+            init_session_iaa(data=annotations, baseline=baseline_text, col=col2)
+
+    # Global statistics
+    if option == "Global project statistics":
+        # User input controllers
+        mode = sidebar.radio("Choose mode to retrieve curated data: ", (
+            "Local directory", "INCEpTION API Host remote"
+        ))
+        data = None
+        if mode == "Local directory":
+            project = sidebar.file_uploader(
+                "Folder that contains curated annotations in XMI 1.1 (.zip format only): ",
+                type="zip"
+            )
+            data = project
+        if mode == "INCEpTION API Host remote":
+            username = sidebar.text_input("Username: ")
+            password = sidebar.text_input("Password: ", type='password')
+            data = (username, password)
+
+        # Validate inputs
+        btn_process = sidebar.button('Process', key='process')
+
+        # Access data with local ressources
+        if btn_process and mode == "Local directory":
+            if data is not None:
                 # create a new session
-                init_session_statistics(remote=True, local=False, data=data)
-            else:
-              st.error("Sorry! Username or Password is empty.")
-
-    # Change data values and visualize new plot
-    if "gs_obj" in st.session_state:
-        if st.session_state["gs_local"] or st.session_state["gs_remote"]:
-            display_data()
-
-
+                init_session_statistics(remote=False, local=True, data=data)
 
+        # Access data with remote ressources
+        if btn_process and mode == "INCEpTION API Host remote":
+            if data is not None:
+                if check_login(username=data[0], password=data[1]):
+                    # create a new session
+                    init_session_statistics(remote=True, local=False, data=data)
+                else:
+                    st.error("Username or Password is empty, please check and retry.")
 
+        # Change data values and visualize new plot
+        if "gs_obj" in st.session_state:
+            if st.session_state["gs_local"] or st.session_state["gs_remote"]:
+                display_data(col1)
 
 
+if __name__ == "__main__":
+    n4a_analytics_dashboard()

datatest/curation_rapid_global.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:804a01b2ffae53103cd67fa51671ccbbbc988cf2796ec40ccb20f1e9283c1b47
-size 4670583

datatest/{test.zip → exemple_IAA_annotations.zip}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab5ced7fa96a8b65ad8077d69761f517b19a57d8ec74e86608101d3bb66c6a54
-size 74199
+oid sha256:1a8058de8efe999f8b2ec4c6162691b4991dbfeee107117f078bdc895c463c6b
+size 91754

n4a_analytics_lib/analytics.py

@@ -1,17 +1,20 @@
 # -*- coding:utf-8 -*-
 
+from itertools import combinations
+from collections import defaultdict, Counter
+
 import pandas as pd
 import seaborn as sns
-import matplotlib
+import matplotlib as plt
 
-matplotlib.use('Agg')
+plt.use('Agg')
 
 import nltk
-
 nltk.download('punkt')
 from nltk.tokenize import sent_tokenize, word_tokenize
 
 from n4a_analytics_lib.project import Project
+from n4a_analytics_lib.metrics_utils import (fleiss_kappa_function, cohen_kappa_function, percentage_agreement_pov)
 
 
 class GlobalStatistics(Project):
@@ -24,7 +27,7 @@ class GlobalStatistics(Project):
 
         self.total_annotations_project = self.df_i['TOTAL'].sum()
 
-    def create_plot(self, type_data):
+    def create_plot(self, type_data: str) -> sns.barplot:
         # apply data filter
         data_tab_filtered = self.df_details.loc[self.df_details['SOURCE_FILE'] == type_data]
         # create a new plot
@@ -37,25 +40,38 @@ class GlobalStatistics(Project):
         return ax.figure
 
 
-
-
-
 class IaaStatistics(Project):
-    def __init__(self, zip_project, baseline_text):
-        super().__init__(zip_project=zip_project, type="iaa")
+    def __init__(self, zip_project, baseline_text, remote=False):
+        super().__init__(zip_project=zip_project, remote=remote, type="iaa")
         self.baseline_text = baseline_text.decode('utf-8')
 
-        # self.docs = {}
-        # self.pairwise = {}
-        # self.similar_mention = []
         self.mentions_per_coder = self.extract_refs(self.annotations, self.annotators, type="mentions")
         self.labels_per_coder = self.extract_refs(self.annotations, self.annotators, type="labels")
-
         self.annotations_per_coders = {coder: dict(zip(ann[1]['mentions'], ann[1]['labels'])) for coder, ann in zip(self.annotators, self.annotations.items())}
+        self.coders_pairs = list(combinations(self.annotations_per_coders, 2))
+        self.similar_mention = list(dict.fromkeys([l for i,j in self.mentions_per_coder.items() for l in j]))
+
+        self.labels_schema = list(dict.fromkeys([label for _, labels in self.labels_per_coder.items() for label in labels]))
 
+        # dataframes and matrix analysis
+        self.base_df = self.build_base_df()
+        self.df_agree = self.base_df [self.base_df[self.annotators].apply(lambda row: self.check_all_equal(row), axis=1)]
+        self.df_disagree = self.base_df[self.base_df[self.annotators].apply(lambda row: self.check_all_not_equal(row), axis=1)]
+        self.coders_matrix = self.base_df.apply(pd.Series.value_counts, 1).fillna(0).astype(int).values
+
+        # totals
+        self.total_annotations = len(self.base_df)
+        self.total_agree = len(self.df_agree)
+        self.total_disagree = len(self.df_disagree)
+
+        # access to metrics
+        self.fleiss_kappa = round(fleiss_kappa_function(self.coders_matrix), 2)
+        self.cohen_kappa_pairs = self.compute_pairs_cohen_kappa()
+        self.percent_agree = percentage_agreement_pov(self.total_agree, self.total_annotations)
+        self.percent_disagree = percentage_agreement_pov(self.total_disagree, self.total_annotations)
 
     @staticmethod
-    def extract_refs(annotations, annotators, type):
+    def extract_refs(annotations: dict, annotators: list, type: str) -> dict:
         return {
             coder: data for coder, ann in zip(
                 annotators,
@@ -63,7 +79,82 @@ class IaaStatistics(Project):
             ) for ref, data in ann[1].items() if ref == type
         }
 
-    def analyze_text(self):
+    @staticmethod
+    def check_all_equal(iterator: list) -> bool:
+        return len(set(iterator)) <= 1
+
+    @staticmethod
+    def check_all_not_equal(iterator: list) -> bool:
+        return len(set(iterator)) > 1
+
+    def plot_confusion_matrix(self, width: int, height: int) -> plt.pyplot.subplots:
+        intermediary = defaultdict(Counter)
+        for (src, tgt), count in self.cohen_kappa_pairs.items():
+            intermediary[src][tgt] = count
+
+        letters = sorted({key for inner in intermediary.values() for key in inner} | set(intermediary.keys()))
+
+        confusion_matrix = [[intermediary[src][tgt] for tgt in letters] for src in letters]
+
+        df_cm = pd.DataFrame(confusion_matrix, letters, letters)
+        mask = df_cm.values == 0
+        sns.set(font_scale=0.7)  # for label size
+        colors = ["#e74c3c", "#f39c12", "#f4d03f", "#5dade2", "#58d68d", "#28b463"]
+
+        fig, ax = plt.pyplot.subplots(figsize=(width, height))
+        sns.heatmap(df_cm, cmap=colors, annot=True, mask=mask, annot_kws={"size": 7}, vmin=0, vmax=1, ax=ax)  # font size
+        return ax
+
+    def build_base_df(self) -> pd.DataFrame:
+        df = pd.DataFrame(self.annotations_per_coders, index=self.similar_mention)
+        for ann in self.annotators:
+            df[ann] = 'None'
+            for mention, value in self.annotations_per_coders[ann].items():
+                df.loc[mention, ann] = value
+        return df
+
+    def compute_pairs_cohen_kappa(self) -> dict:
+        return {
+            (c1, c2): cohen_kappa_function(self.labels_per_coder[c1],
+                                           self.labels_per_coder[c2]) for c1, c2 in self.coders_pairs
+        }
+
+    def count_total_annotations_label(self) -> list:
+        return [
+            (label, self.base_df.astype(object).eq(label).any(1).sum()) for label in self.labels_schema
+        ]
+
+    def total_agree_disagree_per_label(self) -> list:
+        # t[0] : label
+        # t[1] : total_rows_with_label
+        return [(
+            t[0],
+            t[1],
+            (self.base_df[self.base_df.nunique(1).eq(1)].eq(t[0]).any(1).sum() / t[1]) * 100,
+            ((t[1] - self.base_df[self.base_df.nunique(1).eq(1)].eq(t[0]).any(1).sum()) / t[1]) * 100
+        )
+            for t in self.count_total_annotations_label()]
+
+    def plot_agreement_pies(self) -> plt.pyplot.subplots:
+        my_labels = 'agree', 'disagree'
+        my_colors = ['#47DBCD', '#F5B14C']
+        my_explode = (0, 0.1)
+        counter = 0
+        tasks_to_pie = self.total_agree_disagree_per_label()
+        fig, axes = plt.pyplot.subplots(1, len(tasks_to_pie), figsize=(20, 3))
+        for t in tasks_to_pie:
+            tasks = [t[2], t[3]]
+            axes[counter].pie(tasks, autopct='%1.1f%%', startangle=15, shadow=True, colors=my_colors,
+                              explode=my_explode)
+            axes[counter].set_title(t[0])
+            axes[counter].axis('equal')
+            counter += 1
+        fig.set_facecolor("white")
+        fig.legend(labels=my_labels, loc="center right", borderaxespad=0.1, title="Labels alignement")
+        # plt.savefig(f'./out/pie_alignement_labels_{filename_no_extension}.png', dpi=400)
+        return fig
+
+    def analyze_text(self) -> list:
         """returns total sentences, words and characters
         in list format
         """

n4a_analytics_lib/constants.py

@@ -11,4 +11,79 @@ of NER4Archives (Inria/Archives nationales).
 - This tool provides two statistics levels:
     - *Global project statistics*: Analyze named entities in overall curated documents in project;
     - *Inter-Annotator Agreement results*: Analyze results of IAA experiment.  
-"""
+"""
+
+KAPPA_LEGEND = """
+<div>
+	<div id="legend" style="right: 70em;">
+		<h3>🗃 IAA Metrics Legend</h3>
+		<table>
+			<thead>
+				<tr>
+					<th colspan="2">
+						 Kappa interpretation legend 
+					</th>
+				</tr>
+			</thead>
+			<tbody>
+				<tr>
+					<td>
+						 Kappa score (k) 
+					</td>
+					<td>
+						Agreement
+					</td>
+				</tr>
+				<tr style = "background-color: #e74c3c;">
+					<td>
+						 k < 0 
+					</td>
+					<td>
+						 Less chance agreement 
+					</td>
+				</tr>
+				<tr style = "background-color: #f39c12;">
+					<td>
+						 0.01  < k  < 0.20 
+					</td>
+					<td>
+						 Slight agreement 
+					</td>
+				</tr>
+				<tr style = "background-color: #f4d03f;">
+					<td>
+						 0.21  < k  < 0.40 
+					</td>
+					<td>
+						 Fair agreement 
+					</td>
+				</tr>
+				<tr style = "background-color:  #5dade2;">
+					<td>
+						 0.41  < k  < 0.60 
+					</td>
+					<td>
+						 Moderate agreement 
+					</td>
+				</tr>
+				<tr style = "background-color:  #58d68d;">
+					<td>
+						 0.61  < k  < 0.80 
+					</td>
+					<td>
+						 Substantial agreement 
+					</td>
+				</tr>
+				<tr style = "background-color:  #28b463;">
+					<td>
+						 0.81  < k  < 0.99 
+					</td>
+					<td>
+						 Almost perfect agreement 
+					</td>
+				</tr>
+			</tbody>
+		</table>
+	</div>
+</div>
+"""

n4a_analytics_lib/metrics_utils.py

@@ -1,31 +1,52 @@
 # -*- coding:utf-8 -*-
 
+"""Collection of statistics functions.
+"""
+
 import numpy as np
 
-def fleiss_kappa_function(M):
+
+def percentage_agreement_pov(total_pov: int, total_annotations: int) -> float:
+    """Computes a percentage
+    :param total_pov: total agree/disagree annotations
+    :type total_pov: int
+    :param total_annotations:  total annotations in project
+    :type total_annotations: int
+    :rtype: float
+    :return: agreement percentage
+    """
+    return round((total_pov / total_annotations) * 100, 2)
+
+
+def fleiss_kappa_function(matrix: list) -> float:
     """Computes Fleiss' kappa for group of annotators.
-    :param M: a matrix of shape (:attr:'N', :attr:'k') with 'N' = number of subjects and 'k' = the number of categories.
-        'M[i, j]' represent the number of raters who assigned the 'i'th subject to the 'j'th category.
-    :type: numpy matrix
+    :param matrix: a matrix of shape (:attr:'N', :attr:'k') with
+    'N' = number of subjects and 'k' = the number of categories.
+    'M[i, j]' represent the number of raters who assigned
+    the 'i'th subject to the 'j'th category.
+    :type matrix: numpy matrix
     :rtype: float
     :return: Fleiss' kappa score
     """
-    N, k = M.shape  # N is # of items, k is # of categories
-    n_annotators = float(np.sum(M[0, :]))  # # of annotators
+    N, _ = matrix.shape  # N is # of items, k is # of categories
+    n_annotators = float(np.sum(matrix[0, :]))  # # of annotators
     tot_annotations = N * n_annotators  # the total # of annotations
-    category_sum = np.sum(M, axis=0)  # the sum of each category over all items
+    category_sum = np.sum(matrix, axis=0)  # the sum of each category over all items
 
     # chance agreement
     p = category_sum / tot_annotations  # the distribution of each category over all annotations
     PbarE = np.sum(p * p)  # average chance agreement over all categories
 
     # observed agreement
-    P = (np.sum(M * M, axis=1) - n_annotators) / (n_annotators * (n_annotators - 1))
-    Pbar = np.sum(P) / N  # add all observed agreement chances per item and divide by amount of items
+    P = (np.sum(matrix * matrix, axis=1) - n_annotators) / (n_annotators * (n_annotators - 1))
+    Pbar = np.sum(P) / N
+    # add all observed agreement
+    # chances per item and divide by amount of items
 
     return round((Pbar - PbarE) / (1 - PbarE), 4)
 
-def cohen_kappa_function(ann1, ann2):
+
+def cohen_kappa_function(ann1: list, ann2: list) -> float:
     """Computes Cohen kappa for pair-wise annotators.
     :param ann1: annotations provided by first annotator
     :type ann1: list
@@ -50,19 +71,3 @@ def cohen_kappa_function(ann1, ann2):
 
     return round((A - E) / (1 - E), 4)
 
-def interpret_kappa(score):
-    color = ""
-    if score < 0:
-        color= "#e74c3c;"
-    elif 0.01 <= score <= 0.20:
-        color= "#f39c12;"
-    elif 0.21 <= score <= 0.40:
-        color= "#f4d03f;"
-    elif 0.41 <= score <= 0.60:
-        color= "#5dade2;"
-    elif 0.61 <= score <= 0.80:
-        color= "#58d68d;"
-    elif 0.81 <= score <= 0.99:
-        color= "#28b463;"
-
-    return f"<span style='font-size:30px; color: {color}'>{round(score*100, 2)} %</span>"

n4a_analytics_lib/project.py

@@ -1,15 +1,35 @@
 # -*- coding:utf-8 -*-
-import zipfile
+
 from io import BytesIO
 import re
 from zipfile import ZipFile
 import os
 from pathlib import Path
 
-
+import streamlit as st
 from cassis import load_typesystem, load_cas_from_xmi
 
-from n4a_analytics_lib.st_components import st_pb
+
+def st_pb(method):
+    """streamlit decorator to display
+    progress bar
+    """
+    def progress_bar(ref):
+        container = st.empty()
+        bar = st.progress(0)
+        pg_gen = method(ref)
+        try:
+            while True:
+                progress = next(pg_gen)
+                bar.progress(progress[0])
+                if progress[2]:
+                    container.write("✅ Processing... " + progress[1])
+                else:
+                    container.write("❌️ Errror with..." + progress[1])
+        except StopIteration as result:
+            return result.value
+
+    return progress_bar
 
 
 class Project:
@@ -44,8 +64,7 @@ class Project:
         """
         self.annotations = {}
 
-
-        if isinstance(self.zip_project, zipfile.ZipFile) and self.remote and self.type == "global":
+        if isinstance(self.zip_project, ZipFile) and self.remote and self.type == "global":
             for fp in self.zip_project.namelist():
                 if self.typesystem is None:
                     self.typesystem = load_typesystem(BytesIO(self.zip_project.open('TypeSystem.xml').read()))
@@ -53,43 +72,40 @@ class Project:
                     self.documents.append(fp)
                     self.xmi_documents.append(str(self.zip_project.open(fp).read().decode("utf-8")))
 
-
         else:
-         with ZipFile(self.zip_project) as project_zip:
-             if self.type == "global":
-                 regex = re.compile('.*curation/.*/(?!\._).*zip$')
-             elif self.type == "iaa":
-                 regex = re.compile('.*xm[il]$')
-
-             annotation_fps = (fp for fp in project_zip.namelist() if regex.match(fp))
-             for fp in annotation_fps:
-                 if self.type == "global":
-                     with ZipFile(BytesIO(project_zip.read(fp))) as annotation_zip:
-                         if self.typesystem is None:
-                             self.typesystem = load_typesystem(BytesIO(annotation_zip.read('TypeSystem.xml')))
-                         for f in annotation_zip.namelist():
-                             if f.endswith('.xmi'):
-                                 # store source filename
-                                 self.documents.append(Path(fp).parent.name)
-                                 # annotators = []
-                                 # store XMI representation
-                                 self.xmi_documents.append(str(annotation_zip.read(f).decode("utf-8")))
-                 elif self.type == "iaa":
-                     if self.typesystem is None and fp.endswith('.xml'):
-                         self.typesystem = load_typesystem(BytesIO(project_zip.read('TypeSystem.xml')))
-                     else:
-                         if fp.endswith('.xmi'):
-                             # store source filename
-                             self.documents.append(fp)
-                             # set annotators
-                             self.annotators.append(os.path.splitext(fp)[0])
-                             # store XMI representation
-                             self.xmi_documents.append(str(project_zip.read(fp).decode("utf-8")))
-
+            with ZipFile(self.zip_project) as project_zip:
+                if self.type == "global":
+                    regex = re.compile('.*curation/.*/(?!\._).*zip$')
+                elif self.type == "iaa":
+                    regex = re.compile('.*xm[il]$')
+
+                annotation_fps = (fp for fp in project_zip.namelist() if regex.match(fp))
+                for fp in annotation_fps:
+                    if self.type == "global":
+                        with ZipFile(BytesIO(project_zip.read(fp))) as annotation_zip:
+                            if self.typesystem is None:
+                                self.typesystem = load_typesystem(BytesIO(annotation_zip.read('TypeSystem.xml')))
+                            for f in annotation_zip.namelist():
+                                if f.endswith('.xmi'):
+                                    # store source filename
+                                    self.documents.append(Path(fp).parent.name)
+                                    # annotators = []
+                                    # store XMI representation
+                                    self.xmi_documents.append(str(annotation_zip.read(f).decode("utf-8")))
+                    elif self.type == "iaa":
+                        if self.typesystem is None and fp.endswith('.xml'):
+                            self.typesystem = load_typesystem(BytesIO(project_zip.read('TypeSystem.xml')))
+                        else:
+                            if fp.endswith('.xmi'):
+                                # store source filename
+                                self.documents.append(fp)
+                                # set annotators
+                                self.annotators.append(os.path.splitext(fp)[0])
+                                # store XMI representation
+                                self.xmi_documents.append(str(project_zip.read(fp).decode("utf-8")))
 
         self.extract_ne()
 
-
     @st_pb
     def extract_ne(self):
         count = 0
@@ -112,4 +128,3 @@ class Project:
 
 
 
-

n4a_analytics_lib/st_components.py

@@ -1,22 +1,198 @@
 # -*- coding:utf-8 -*-
 
+import io
+
+import pandas
 import streamlit as st
+from pycaprio import Pycaprio, mappings
+from zipfile import ZipFile
+from requests.exceptions import JSONDecodeError
+
+from n4a_analytics_lib.analytics import (GlobalStatistics,
+                                         IaaStatistics)
+from n4a_analytics_lib.constants import KAPPA_LEGEND
+
+
+@st.cache
+def convert_df(df_ex: pandas.DataFrame) -> bytes:
+    return df_ex.to_csv(encoding="utf-8").encode('utf-8')
+
+
+def check_login(username: str, password: str) -> bool:
+    if (len(username) == 0) or (len(password) == 0):
+        return False
+    return True
+
+
+def display_data(col: st.columns) -> None:
+    col.metric("Total curated annotations",
+                f"{st.session_state['gs_obj'].total_annotations_project} Named entities")
+    col.dataframe(st.session_state['gs_obj'].df_i)
+    selected_data = col.selectbox('Select specific data to display bar plot:',
+                                   st.session_state['gs_obj'].documents, key="selector_data")
+    col.pyplot(st.session_state['gs_obj'].create_plot(selected_data))
+
+
+def template_agreement_dataframe(title: str,
+                                 df: pandas.DataFrame,
+                                 total_pov: int,
+                                 total_annotations: int,
+                                 percentage_pov: float,
+                                 mode: str) -> None:
+    st.subheader(title)
+    st.markdown(f"{total_pov} / {total_annotations} annotations ({percentage_pov} %)")
+    st.download_button(
+            "Press to Download CSV",
+            convert_df(df),
+            f"csv_annotators_{mode}.csv",
+            "text/csv",
+            key=f'download-csv_{mode}'
+        )
+    st.dataframe(df)
+
+
+def init_session_iaa(data: st.file_uploader,
+                     baseline: st.file_uploader,
+                     col: st.columns) -> None:
+    project_analyzed = IaaStatistics(zip_project=data, baseline_text=baseline.getvalue())
+    baseline_analyzer = project_analyzed.analyze_text()
+
+    col.markdown(f"""
+           ### BASELINE TEXT: {baseline.name}
+
+            - sentences:  {baseline_analyzer[0]}
+            - words: {baseline_analyzer[1]}
+            - characters: {baseline_analyzer[2]}
+           """)
+
+    st.markdown("## 📈 IAA metrics")
+    col1_kappa, col2_kappa = st.columns(2)
+
+    # Display Kappa group
+    col1_kappa.subheader("Fleiss Kappa (global score for group):")
+    col1_kappa.markdown(interpret_kappa(project_analyzed.fleiss_kappa), unsafe_allow_html=True)
+
+    # Display pairs kappa
+    col1_kappa.subheader("Cohen Kappa (score for annotators pair):")
+    for coders, c_k in project_analyzed.compute_pairs_cohen_kappa().items():
+        col1_kappa.markdown(f"* {coders[0]} <> {coders[1]} : {interpret_kappa(c_k)}", unsafe_allow_html=True)
+
+    # Display Kappa legend
+    col2_kappa.markdown(KAPPA_LEGEND, unsafe_allow_html=True)
+
+    # Plot confusion matrix
+    if st.checkbox('Display confusion matrix'):
+        width = st.slider("matrix width", 1, 10, 14)
+        height = st.slider("matrix height", 1, 10, 4)
+        st.pyplot(project_analyzed.plot_confusion_matrix(width=width, height=height).figure)
+
+    # Agree CSV
+    template_agreement_dataframe(title="✅️ Agree annotations",
+                                 df=project_analyzed.df_agree,
+                                 total_pov=project_analyzed.total_agree,
+                                 total_annotations=project_analyzed.total_annotations,
+                                 percentage_pov=project_analyzed.percent_agree,
+                                 mode="agree")
+    # Disagree CSV
+    template_agreement_dataframe(title="❌ Disagree annotations",
+                                 df=project_analyzed.df_disagree,
+                                 total_pov=project_analyzed.total_disagree,
+                                 total_annotations=project_analyzed.total_annotations,
+                                 percentage_pov=project_analyzed.percent_disagree,
+                                 mode="disagree")
+    # Pie plot
+    st.subheader("🏷️ Global Labels Statistics")
+    st.pyplot(project_analyzed.plot_agreement_pies().figure)
 
 
-def st_pb(method):
-    def progress_bar(ref):
-        container = st.empty()
-        bar = st.progress(0)
-        pg_gen = method(ref)
+def init_session_statistics(remote: bool, local: bool, data: tuple) -> None:
+    # clear session
+    st.session_state = {}
+
+    # create a session variable
+    st.session_state["gs_local"] = local
+    st.session_state["gs_remote"] = remote
+
+    # create a new object:
+    # if remote fetch data from API Host first
+    if remote and not(local):
+        st.success('Fetch curated documents from host INCEpTION API in progress...')
         try:
-            while True:
-                progress = next(pg_gen)
-                bar.progress(progress[0])
-                if progress[2]:
-                    container.write("✅ Processing... " + progress[1])
-                else:
-                    container.write("❌️ Errror with..." + progress[1])
-        except StopIteration as result:
-            return result.value
-
-    return progress_bar
+            fetch_curated_data_from_remote(
+                username=data[0],
+                password=data[1]
+        )
+        except JSONDecodeError:
+            # username / password incorrect
+            st.error('Username or Password is incorrect please retry.')
+            st.session_state = {}
+
+    if local and not(remote):
+        st.session_state["gs_obj"] = GlobalStatistics(zip_project=data, remote=False)
+
+
+def fetch_curated_data_from_remote(username: str,
+                                   password: str,
+                                   endpoint: str = "https://inception.dhlab.epfl.ch/prod",
+                                   project_title: str = "ner4archives-template") -> None:
+    # open a client
+    client = Pycaprio(inception_host=endpoint, authentication=(str(username), str(password)))
+
+    # get project object
+    project_name = [p for p in client.api.projects() if p.project_name == project_title]
+
+    # get all documents from project
+    documents = client.api.documents(project_name[0].project_id)
+
+    curations = []
+    zipfiles = []
+    count = 0
+    flag = "a"
+    # iterate over all documents and retrieve only curated into ZIP container
+    for document in documents:
+        if count > 0:
+            flag = "r"
+        if document.document_state == mappings.DocumentState.CURATION_COMPLETE:
+            curated_content = client.api.curation(project_name[0].project_id, document,
+                                                  curation_format=mappings.InceptionFormat.UIMA_CAS_XMI_XML_1_1)
+            curations.append(curated_content)
+            for curation in curations:
+                z = ZipFile(io.BytesIO(curation), mode=flag)
+                zipfiles.append(z)
+
+        count += 1
+
+    # Merge all zip in one
+    with zipfiles[0] as z1:
+        for fname in zipfiles[1:]:
+            zf = fname
+            # print(zf.namelist())
+            for n in zf.namelist():
+                if n not in z1.namelist():
+                    z1.writestr(n, zf.open(n).read())
+
+        # Create a new object
+        st.session_state["gs_obj"] = GlobalStatistics(zip_project=z1, remote=True)
+
+
+def interpret_kappa(score: float) -> str:
+    color = ""
+    if score < 0:
+        color= "#e74c3c;"
+    elif 0.01 <= score <= 0.20:
+        color= "#f39c12;"
+    elif 0.21 <= score <= 0.40:
+        color= "#f4d03f;"
+    elif 0.41 <= score <= 0.60:
+        color= "#5dade2;"
+    elif 0.61 <= score <= 0.80:
+        color= "#58d68d;"
+    elif 0.81 <= score <= 0.99:
+        color= "#28b463;"
+
+    return f"<span style='font-size:30px; color: {color}'>{round(score*100, 2)} %</span>"
+
+
+
+
+