Datasets documentation

.ipynb_checkpoints/Model visualisation-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

DataSets.md

@@ -0,0 +1,17 @@
+#Datasets
+
+We are planning to use the following datasets to train the models.
+
+##Base Model Training
+
+[The British National Corpus](http://www.natcorp.ox.ac.uk/)
+
+##Question Answering
+
+##Reasoning
+
+[Avicenna: Syllogistic Commonsense Reasoning](https://github.com/ZeinabAghahadi/Syllogistic-Commonsense-Reasoning)
+
+##Consistency
+
+[Stanford Natural Language Inference Corpus](https://www.kaggle.com/datasets/stanfordu/stanford-natural-language-inference-corpus)

qarac/models/layers/HyenaLayer.py

@@ -11,6 +11,7 @@ import keras_nlp
 import tensorflow
 import warnings
 
+@tensorflow.function
 def convolve(x,y):
     
     fx = tensorflow.vectorized_map(fft, x, warn=False)
@@ -54,6 +55,7 @@ class HyenaLayer(keras.layers.Layer):
         self.data_projection = None
         self.filters = None
     
+    @tensorflow.function
     def positional_encoding(self,X):
         t = tensorflow.dtypes.saturate_cast(tensorflow.ragged.range(X.row_lengths()),
                                             tensorflow.float32)
@@ -90,7 +92,7 @@ class HyenaLayer(keras.layers.Layer):
             x = concat(x,tensorflow.zeros_like(x))
             f = concat(f,tensorflow.zeros_like(f))
         y = x[:,:,:,0]
-        for i in range(self.stages):
+        for i in tensorflow.range(self.stages):
             y = convolve(y,f[:,:,:,i])*x[:,:,:,i+1]
         if self.causal:
             for (i,n) in enumerate(X.row_lengths()):

scripts.py

@@ -30,7 +30,8 @@ def train_base_model(task,filename):
     model.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics='accuracy')
     model.fit(train_data,
               epochs=100,
-              workers = 16)
+              workers = 16,
+              use_multiprocessing=True)
     test_data=qarac.corpora.Batcher.Batcher(test)
     print(model.evaluate(test_data))
     model.save(filename)