Upload folder using huggingface_hub

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,6 @@
+[cache]
+    dir = /data/dvc-cache/FintoAI-data-KAUNO
+    shared = group
+    type = symlink
+[core]
+    autostage = true

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.github/workflows/huggingfacehub-sync.yml

@@ -0,0 +1,31 @@
+name: Sync with Hugging Face Hub
+on:
+  #push:
+  #  branches:
+  #  - main
+  workflow_dispatch
+jobs:
+  sync-to-hfhub:
+    name: Sync to Hugging Face Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Sync to Hugging Face Hub
+        uses: juhoinkinen/huggingface-sync-action@v0.0.10
+        with:
+          # The Hugging Face repo id you want to sync to. (ex. 'username/reponame')
+          # A repo with this name will be created if it doesn't exist. Required.
+          huggingface_repo_id: 'juhoinkinen/FintoAI-data-KAUNO-3'
+        
+          # Hugging Face token with write access. Required.
+          # Here, we provide a token that we called `HF_TOKEN` when we added the secret to our GitHub repo.
+          hf_token: ${{ secrets.HF_TOKEN }}
+        
+          # The type of repo you are syncing to: model, dataset, or space.
+          # Defaults to space.
+          repo_type: 'model'
+          
+          # If true and the Hugging Face repo doesn't already exist, it will be created
+          # as a private repo.
+          #
+          # Note: this param has no effect if the repo already exists.
+          private: true

.gitignore

@@ -0,0 +1,2 @@
+venv
+venv-installed

LICENSE

@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

README.md

@@ -0,0 +1,38 @@
+---
+license: cc0-1.0
+language:
+- fi
+pipeline_tag: text-classification
+thumbnail: https://raw.githubusercontent.com/NatLibFi/FintoAI/main/ai.finto.fi/static/img/finto-ai-social.png
+tags:
+- glam
+- lam
+- subject indexing
+- annif
+---
+# FintoAI-data-KAUNO
+The Annif projects with KAUNO vocabulary used at the [Finto AI service](https://ai.finto.fi/).
+
+This repository is mirrored from GitHub to the 🤗 Hugging Face Hub; 
+the GitHub repository does not contain the model files, but only the configurations for maintaining the projects, see below.
+
+## Models
+The data files for projects and vocabularies are stored in the
+[`/data`](https://huggingface.co/juhoinkinen/FintoAI-data-KAUNO/tree/main/data)
+directory of this repository in the 🤗 Hugging Face Hub.
+
+## DVC pipeline
+The projects are trained and evaluated using a [DVC (Data Version Control) pipeline](https://dvc.org/doc/start/data-management/data-pipelines) defined in [dvc.yaml](./dvc.yaml).
+
+The pipeline takes care of 
+
+1. installing Annif in a venv,
+2. loading the vocabulary,
+3. training the projects,
+4. evaluating the projects.
+
+When the necessary vocabulary and training corpora are in place the pipeline can be run using the command
+
+    dvc repro
+    
+For more information about using DVC with Annif projects see the [DVC exercise of Annif tutorial](https://github.com/NatLibFi/Annif-tutorial/blob/master/exercises/OPT_dvc.md).

corpora/.gitignore

@@ -0,0 +1,2 @@
+/kauno-skos.ttl
+/kauno-skos-reduced.ttl

corpora/finna/.gitignore

@@ -0,0 +1 @@
+/kauno-finna-fin.tsv.gz

corpora/finna/kauno-finna-fin.tsv.gz.dvc

@@ -0,0 +1,10 @@
+md5: dae92ef1d2bed2a3c21e7631b6cdcb5e
+deps:
+- md5: ae570f73cbc72bd952efebd67575cda4
+  size: 63666506
+  path: /data/Annif-corpora/training/kauno-finna-fin.tsv.gz
+  hash: md5
+outs:
+- md5: ae570f73cbc72bd952efebd67575cda4
+  size: 63666506
+  path: kauno-finna-fin.tsv.gz

corpora/kauno-skos-reduced.ttl.dvc

@@ -0,0 +1,11 @@
+md5: 4d29a28b7156dd12c51a8626e9f882bf
+deps:
+- md5: 6ca997fbaed020b9bfa867c991b19739
+  size: 17501880
+  path: /data/Annif-corpora-restricted/kirjasampo/kauno-skos-reduced.ttl
+  hash: md5
+outs:
+- md5: 6ca997fbaed020b9bfa867c991b19739
+  size: 17501880
+  path: kauno-skos-reduced.ttl
+  hash: md5

corpora/kirjasampo/.gitignore

@@ -0,0 +1,3 @@
+/kirjasampo-kauno-train.tsv
+/kirjasampo-kauno-validate.tsv
+/kirjasampo-kauno-test.tsv

corpora/kirjasampo/kirjasampo-kauno-test.tsv.dvc

@@ -0,0 +1,11 @@
+md5: cca190c38bb6b6fbdeb2c4ca5c85abd9
+deps:
+- md5: 2a491f3d2b35df602a3730db99f7605c
+  size: 1502390
+  path: /data/Annif-corpora-restricted/kirjasampo/kirjasampo-kauno-test.tsv
+  hash: md5
+outs:
+- md5: 2a491f3d2b35df602a3730db99f7605c
+  size: 1502390
+  path: kirjasampo-kauno-test.tsv
+  hash: md5

corpora/kirjasampo/kirjasampo-kauno-train.tsv.dvc

@@ -0,0 +1,11 @@
+md5: a59acca5aa93619ecdebc3083d0377a4
+deps:
+- md5: 77c01592d2c7c34f04bbee75b6584ec8
+  size: 58959362
+  path: /data/Annif-corpora-restricted/kirjasampo/kirjasampo-kauno-train.tsv
+  hash: md5
+outs:
+- md5: 77c01592d2c7c34f04bbee75b6584ec8
+  size: 58959362
+  path: kirjasampo-kauno-train.tsv
+  hash: md5

corpora/kirjasampo/kirjasampo-kauno-validate.tsv.dvc

@@ -0,0 +1,11 @@
+md5: 54798c42f7f1117903899095aa02821e
+deps:
+- md5: 6dbd774820c16dbd440da71151e36079
+  size: 2455221
+  path: /data/Annif-corpora-restricted/kirjasampo/kirjasampo-kauno-validate.tsv
+  hash: md5
+outs:
+- md5: 6dbd774820c16dbd440da71151e36079
+  size: 2455221
+  path: kirjasampo-kauno-validate.tsv
+  hash: md5

data/projects/.gitignore

@@ -0,0 +1,8 @@
+/kauno-mllm-fi
+/kauno-parabel-fi
+/kauno-bonsai-fi
+/kauno-ks-parabel-fi
+/kauno-finna-bonsai-fi
+/kauno-finna-parabel-fi
+/kauno-ks-bonsai-fi
+/kauno-fi

data/vocabs/.gitignore

@@ -0,0 +1 @@
+/kauno

dvc.lock

@@ -0,0 +1,542 @@
+schema: '2.0'
+stages:
+  install:
+    cmd:
+    - python3 -m venv venv
+    - . venv/bin/activate && pip install -U pip wheel setuptools && pip install -r
+      requirements.txt
+    - cp requirements.txt venv-installed
+    deps:
+    - path: requirements.txt
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+  loadvoc:
+    cmd: venv/bin/annif load-vocab --force kauno corpora/kauno-skos-reduced.ttl
+    deps:
+    - path: corpora/kauno-skos-reduced.ttl
+      hash: md5
+      md5: 6ca997fbaed020b9bfa867c991b19739
+      size: 17501880
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: data/vocabs/kauno
+      hash: md5
+      md5: 8d23b787082ad96201ffb484ec6fb681.dir
+      size: 34076325
+      nfiles: 3
+  train-mllm:
+    cmd: venv/bin/annif train kauno-mllm-fi -j 8 -d 2000 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-train.tsv
+      hash: md5
+      md5: 77c01592d2c7c34f04bbee75b6584ec8
+      size: 58959362
+    - path: data/vocabs/kauno
+      hash: md5
+      md5: 8d23b787082ad96201ffb484ec6fb681.dir
+      size: 34076325
+      nfiles: 3
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: data/projects/kauno-mllm-fi
+      hash: md5
+      md5: 23c57a221af59d0d2541a7df9e37ad7c.dir
+      size: 2261733
+      nfiles: 2
+  train-omikuji@parabel:
+    cmd: venv/bin/annif train kauno-parabel-fi -j 8 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-train.tsv
+      md5: 6200fab3e9179186787cd49dbcaa587a
+      size: 39921230
+    - path: data/vocabs/kauno
+      md5: 43f49eabb66648de35d01d57d196e196.dir
+      size: 33952771
+      nfiles: 3
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: data/projects/kauno-parabel-fi
+      md5: ba358be8ebda5347eba4825fcebcf43a.dir
+      size: 327572347
+      nfiles: 6
+  train-omikuji@bonsai:
+    cmd: venv/bin/annif train kauno-bonsai-fi -j 8 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-train.tsv
+      md5: 6200fab3e9179186787cd49dbcaa587a
+      size: 39921230
+    - path: data/vocabs/kauno
+      md5: 43f49eabb66648de35d01d57d196e196.dir
+      size: 33952771
+      nfiles: 3
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: data/projects/kauno-bonsai-fi
+      md5: 93b109deb653d12ea6137f3f67742934.dir
+      size: 428545588
+      nfiles: 6
+  eval@bonsai:
+    cmd:
+    - venv/bin/annif eval kauno-bonsai-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-bonsai-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      md5: 48611b60c8006116b9c8686456dcfbf6
+      size: 1535757
+    - path: data/projects/kauno-bonsai-fi
+      md5: 93b109deb653d12ea6137f3f67742934.dir
+      size: 428545588
+      nfiles: 6
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: reports/kauno-bonsai-fi.json
+      md5: 0cb706d29e2c34c41292f422c0d71dd6
+      size: 93
+  eval@mllm:
+    cmd:
+    - venv/bin/annif eval kauno-mllm-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-mllm-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-mllm-fi
+      hash: md5
+      md5: 23c57a221af59d0d2541a7df9e37ad7c.dir
+      size: 2261733
+      nfiles: 2
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-mllm-fi.json
+      hash: md5
+      md5: 9b94207439de0dee3c8efb58eaf23bdb
+      size: 95
+  eval@parabel:
+    cmd:
+    - venv/bin/annif eval kauno-parabel-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-parabel-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      md5: 48611b60c8006116b9c8686456dcfbf6
+      size: 1535757
+    - path: data/projects/kauno-parabel-fi
+      md5: ba358be8ebda5347eba4825fcebcf43a.dir
+      size: 327572347
+      nfiles: 6
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: reports/kauno-parabel-fi.json
+      md5: 56697a462498d0fc1b0dbe0f39dce592
+      size: 94
+  eval@ensemble:
+    cmd:
+    - venv/bin/annif eval kauno-ensemble-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-ensemble-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-ensemble-fi
+      hash: md5
+      md5: 3c5b9e385b3d59c658f823a92dfd34f6.dir
+      size: 1223621179
+      nfiles: 3
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-ensemble-fi.json
+      hash: md5
+      md5: bc2d8f32c841e8941bba9fcd88a92896
+      size: 94
+  train-omikuji-ks@parabel:
+    cmd: venv/bin/annif train kauno-ks-parabel-fi -j 8 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-train.tsv
+      md5: 6200fab3e9179186787cd49dbcaa587a
+      size: 39921230
+    - path: data/vocabs/kauno
+      md5: 43f49eabb66648de35d01d57d196e196.dir
+      size: 33952771
+      nfiles: 3
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: data/projects/kauno-ks-parabel-fi
+      md5: d24263f3ad1b5c3373ba6b1b86a4a222.dir
+      size: 327667525
+      nfiles: 6
+  train-omikuji-finna@bonsai:
+    cmd: venv/bin/annif train kauno-finna-bonsai-fi -j 8 corpora/finna/kauno-finna-fin.tsv.gz
+    deps:
+    - path: corpora/finna/kauno-finna-fin.tsv.gz
+      md5: ae570f73cbc72bd952efebd67575cda4
+      size: 63666506
+    - path: data/vocabs/kauno
+      md5: 43f49eabb66648de35d01d57d196e196.dir
+      size: 33952771
+      nfiles: 3
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: data/projects/kauno-finna-bonsai-fi
+      md5: 16418f34c02a34155b9db383138a8d87.dir
+      size: 546792457
+      nfiles: 6
+  train-omikuji-finna@parabel:
+    cmd: venv/bin/annif train kauno-finna-parabel-fi -j 8 corpora/finna/kauno-finna-fin.tsv.gz
+    deps:
+    - path: corpora/finna/kauno-finna-fin.tsv.gz
+      md5: ae570f73cbc72bd952efebd67575cda4
+      size: 63666506
+    - path: data/vocabs/kauno
+      md5: 43f49eabb66648de35d01d57d196e196.dir
+      size: 33952771
+      nfiles: 3
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: data/projects/kauno-finna-parabel-fi
+      md5: f952af490b34403547b9dd3204d63e15.dir
+      size: 379427399
+      nfiles: 6
+  eval@finna-bonsai:
+    cmd:
+    - venv/bin/annif eval kauno-finna-bonsai-fi -j 8 -m F1@5 -m NDCG --metrics-file
+      reports/kauno-finna-bonsai-fi.json corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-finna-bonsai-fi
+      hash: md5
+      md5: eb1d5d6d196c0237832da37d53680b78.dir
+      size: 542003428
+      nfiles: 6
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-finna-bonsai-fi.json
+      hash: md5
+      md5: fe5bc33b1f952c4d330ee1a12613fd9c
+      size: 94
+  eval@finna-parabel:
+    cmd:
+    - venv/bin/annif eval kauno-finna-parabel-fi -j 8 -m F1@5 -m NDCG --metrics-file
+      reports/kauno-finna-parabel-fi.json corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      md5: 48611b60c8006116b9c8686456dcfbf6
+      size: 1535757
+    - path: data/projects/kauno-finna-parabel-fi
+      md5: f952af490b34403547b9dd3204d63e15.dir
+      size: 379427399
+      nfiles: 6
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: reports/kauno-finna-parabel-fi.json
+      md5: e037704630021469933d06952c18d27e
+      size: 95
+  eval@ks-parabel:
+    cmd:
+    - venv/bin/annif eval kauno-ks-parabel-fi -j 8 -m F1@5 -m NDCG --metrics-file
+      reports/kauno-ks-parabel-fi.json corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      md5: 48611b60c8006116b9c8686456dcfbf6
+      size: 1535757
+    - path: data/projects/kauno-ks-parabel-fi
+      md5: d24263f3ad1b5c3373ba6b1b86a4a222.dir
+      size: 327667525
+      nfiles: 6
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: reports/kauno-ks-parabel-fi.json
+      md5: 5615be39b149e641186f32916d80b0a5
+      size: 94
+  train-omikuji-ks@bonsai:
+    cmd: venv/bin/annif train kauno-ks-bonsai-fi -j 8 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-train.tsv
+      md5: 6200fab3e9179186787cd49dbcaa587a
+      size: 39921230
+    - path: data/vocabs/kauno
+      md5: 43f49eabb66648de35d01d57d196e196.dir
+      size: 33952771
+      nfiles: 3
+    - path: venv-installed
+      md5: b1b83339d2df42e93f3df0e83b1e85ab
+      size: 42
+    outs:
+    - path: data/projects/kauno-ks-bonsai-fi
+      md5: 3130bd53059af8c55630d94116657d4f.dir
+      size: 426174615
+      nfiles: 6
+  eval@ks-bonsai:
+    cmd:
+    - venv/bin/annif eval kauno-ks-bonsai-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-ks-bonsai-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-ks-bonsai-fi
+      hash: md5
+      md5: 7f3dae80e066dae4428a86bba7eea413.dir
+      size: 745367632
+      nfiles: 6
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-ks-bonsai-fi.json
+      hash: md5
+      md5: 9c16b8188723d2a92c8a4117e841fa83
+      size: 94
+  train-omikuji-ks:
+    cmd: venv/bin/annif train kauno-ks-bonsai-fi -j 8 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-train.tsv
+      hash: md5
+      md5: 77c01592d2c7c34f04bbee75b6584ec8
+      size: 58959362
+    - path: data/vocabs/kauno
+      hash: md5
+      md5: 8d23b787082ad96201ffb484ec6fb681.dir
+      size: 34076325
+      nfiles: 3
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: data/projects/kauno-ks-bonsai-fi
+      hash: md5
+      md5: 7f3dae80e066dae4428a86bba7eea413.dir
+      size: 745367632
+      nfiles: 6
+  train-omikuji-finna:
+    cmd: venv/bin/annif train kauno-finna-bonsai-fi -j 8 corpora/finna/kauno-finna-fin.tsv.gz
+    deps:
+    - path: corpora/finna/kauno-finna-fin.tsv.gz
+      md5: ae570f73cbc72bd952efebd67575cda4
+      size: 63666506
+    - path: data/vocabs/kauno
+      hash: md5
+      md5: 8d23b787082ad96201ffb484ec6fb681.dir
+      size: 34076325
+      nfiles: 3
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: data/projects/kauno-finna-bonsai-fi
+      hash: md5
+      md5: eb1d5d6d196c0237832da37d53680b78.dir
+      size: 542003428
+      nfiles: 6
+  train-ensemble:
+    cmd: venv/bin/annif train -j 8 kauno-ensemble-fi corpora/kirjasampo/kirjasampo-kauno-validate.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-validate.tsv
+      hash: md5
+      md5: 6dbd774820c16dbd440da71151e36079
+      size: 2455221
+    - path: data/projects/kauno-finna-bonsai-fi
+      hash: md5
+      md5: eb1d5d6d196c0237832da37d53680b78.dir
+      size: 542003428
+      nfiles: 6
+    - path: data/projects/kauno-ks-bonsai-fi
+      hash: md5
+      md5: 7f3dae80e066dae4428a86bba7eea413.dir
+      size: 745367632
+      nfiles: 6
+    - path: data/projects/kauno-mllm-fi
+      hash: md5
+      md5: 23c57a221af59d0d2541a7df9e37ad7c.dir
+      size: 2261733
+      nfiles: 2
+    - path: data/vocabs/kauno
+      hash: md5
+      md5: 8d23b787082ad96201ffb484ec6fb681.dir
+      size: 34076325
+      nfiles: 3
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: data/projects/kauno-ensemble-fi
+      hash: md5
+      md5: 3c5b9e385b3d59c658f823a92dfd34f6.dir
+      size: 1223621179
+      nfiles: 3
+  train-nn-ensemble:
+    cmd: venv/bin/annif train -j 8 kauno-fi corpora/kirjasampo/kirjasampo-kauno-validate.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-validate.tsv
+      hash: md5
+      md5: 6dbd774820c16dbd440da71151e36079
+      size: 2455221
+    - path: data/projects/kauno-finna-bonsai-fi
+      hash: md5
+      md5: eb1d5d6d196c0237832da37d53680b78.dir
+      size: 542003428
+      nfiles: 6
+    - path: data/projects/kauno-ks-bonsai-fi
+      hash: md5
+      md5: 7f3dae80e066dae4428a86bba7eea413.dir
+      size: 745367632
+      nfiles: 6
+    - path: data/projects/kauno-mllm-fi
+      hash: md5
+      md5: 23c57a221af59d0d2541a7df9e37ad7c.dir
+      size: 2261733
+      nfiles: 2
+    - path: data/vocabs/kauno
+      hash: md5
+      md5: 8d23b787082ad96201ffb484ec6fb681.dir
+      size: 34076325
+      nfiles: 3
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: data/projects/kauno-fi
+      hash: md5
+      md5: ce989b5df6c6771b0e60aefb8a7d4ec9.dir
+      size: 1223621179
+      nfiles: 3
+  eval@mllm-fi:
+    cmd:
+    - venv/bin/annif eval kauno-mllm-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-mllm-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-mllm-fi
+      hash: md5
+      md5: 23c57a221af59d0d2541a7df9e37ad7c.dir
+      size: 2261733
+      nfiles: 2
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-mllm-fi.json
+      hash: md5
+      md5: 9b94207439de0dee3c8efb58eaf23bdb
+      size: 95
+  eval@ks-bonsai-fi:
+    cmd:
+    - venv/bin/annif eval kauno-ks-bonsai-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-ks-bonsai-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-ks-bonsai-fi
+      hash: md5
+      md5: 7f3dae80e066dae4428a86bba7eea413.dir
+      size: 745367632
+      nfiles: 6
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-ks-bonsai-fi.json
+      hash: md5
+      md5: 9c16b8188723d2a92c8a4117e841fa83
+      size: 94
+  eval@finna-bonsai-fi:
+    cmd:
+    - venv/bin/annif eval kauno-finna-bonsai-fi -j 8 -m F1@5 -m NDCG --metrics-file
+      reports/kauno-finna-bonsai-fi.json corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-finna-bonsai-fi
+      hash: md5
+      md5: eb1d5d6d196c0237832da37d53680b78.dir
+      size: 542003428
+      nfiles: 6
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-finna-bonsai-fi.json
+      hash: md5
+      md5: fe5bc33b1f952c4d330ee1a12613fd9c
+      size: 94
+  eval@fi:
+    cmd:
+    - venv/bin/annif eval kauno-fi -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-fi.json
+      corpora/kirjasampo/kirjasampo-kauno-test.tsv
+    deps:
+    - path: corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      hash: md5
+      md5: 2a491f3d2b35df602a3730db99f7605c
+      size: 1502390
+    - path: data/projects/kauno-fi
+      hash: md5
+      md5: ce989b5df6c6771b0e60aefb8a7d4ec9.dir
+      size: 1223621179
+      nfiles: 3
+    - path: venv-installed
+      hash: md5
+      md5: f85f21b68735b126c2241fbff83fd0ef
+      size: 41
+    outs:
+    - path: reports/kauno-fi.json
+      hash: md5
+      md5: 500d93c3d47823dbef945b2efa8899e4
+      size: 93

dvc.yaml

@@ -0,0 +1,76 @@
+stages:
+  # Ensure Annif is installed
+  install:
+    cmd:
+    - python3 -m venv venv
+    - . venv/bin/activate && pip install -U pip wheel setuptools && pip install -r requirements.txt
+    - cp requirements.txt venv-installed
+    deps:
+    - requirements.txt
+    outs:
+    - venv-installed:
+        cache: false
+  # Load KAUNO vocabulary
+  loadvoc:
+    cmd: venv/bin/annif load-vocab --force kauno corpora/kauno-skos-reduced.ttl
+    deps:
+    - venv-installed
+    - corpora/kauno-skos-reduced.ttl
+    outs:
+    - data/vocabs/kauno
+  # Train MLLM project
+  train-mllm:
+    cmd: venv/bin/annif train kauno-mllm-fi -j 8 -d 2000 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - venv-installed
+    - corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    - data/vocabs/kauno
+    outs:
+    - data/projects/kauno-mllm-fi
+  # Train Omikuji project using Kirjasampo data
+  train-omikuji-ks:
+    cmd: venv/bin/annif train kauno-ks-bonsai-fi -j 8 corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    deps:
+    - venv-installed
+    - corpora/kirjasampo/kirjasampo-kauno-train.tsv
+    - data/vocabs/kauno
+    outs:
+    - data/projects/kauno-ks-bonsai-fi
+  # Train Omikuji project using Finna data
+  train-omikuji-finna:
+    cmd: venv/bin/annif train kauno-finna-bonsai-fi -j 8 corpora/finna/kauno-finna-fin.tsv.gz
+    deps:
+    - venv-installed
+    - corpora/finna/kauno-finna-fin.tsv.gz
+    - data/vocabs/kauno
+    outs:
+    - data/projects/kauno-finna-bonsai-fi
+  # Train NN ensemble
+  train-nn-ensemble:
+    cmd: venv/bin/annif train -j 8 kauno-fi corpora/kirjasampo/kirjasampo-kauno-validate.tsv
+    deps:
+    - venv-installed
+    - corpora/kirjasampo/kirjasampo-kauno-validate.tsv
+    - data/vocabs/kauno
+    - data/projects/kauno-mllm-fi
+    - data/projects/kauno-ks-bonsai-fi
+    - data/projects/kauno-finna-bonsai-fi
+    outs:
+    - data/projects/kauno-fi
+  # Evaluate projects
+  eval:
+    foreach:
+      - mllm-fi
+      - ks-bonsai-fi
+      - finna-bonsai-fi
+      - fi
+    do:
+      cmd:
+      - venv/bin/annif eval kauno-${item} -j 8 -m F1@5 -m NDCG --metrics-file reports/kauno-${item}.json corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      deps:
+      - venv-installed
+      - corpora/kirjasampo/kirjasampo-kauno-test.tsv
+      - data/projects/kauno-${item}
+      metrics:
+      - reports/kauno-${item}.json:
+          cache: false

projects.d/2-projects-kauno.toml

@@ -0,0 +1,44 @@
+# KAUNO Finnish
+
+[kauno-mllm-fi]
+name="KAUNO MLLM Finnish"
+language="fi"
+backend="mllm"
+analyzer="voikko(fi)"
+vocab="kauno"
+access="hidden"
+
+[kauno-ks-bonsai-fi]
+name="KAUNO KS Omikuji Bonsai Finnish"
+language="fi"
+backend="omikuji"
+analyzer="voikko(fi)"
+vocab="kauno"
+cluster_balanced=false
+cluster_k=100
+max_depth=3
+min_df=2
+ngram=2
+transform="limit(5000)"
+access="hidden"
+
+[kauno-finna-bonsai-fi]
+name="KAUNO Finna Omikuji Bonsai Finnish"
+language="fi"
+backend="omikuji"
+analyzer="voikko(fi)"
+vocab="kauno"
+cluster_balanced=false
+cluster_k=100
+max_depth=3
+min_df=2
+ngram=2
+transform="limit(5000)"
+access="hidden"
+
+[kauno-fi]
+name="KAUNO suomi"
+language="fi"
+backend="nn_ensemble"
+sources="kauno-mllm-fi:0.1293,kauno-ks-bonsai-fi:0.5518,kauno-finna-bonsai-fi:0.3189"
+vocab="kauno"

projects.toml

@@ -0,0 +1,44 @@
+# KAUNO Finnish
+
+[kauno-mllm-fi]
+name="KAUNO MLLM Finnish"
+language="fi"
+backend="mllm"
+analyzer="voikko(fi)"
+vocab="kauno"
+access="hidden"
+
+[kauno-ks-bonsai-fi]
+name="KAUNO KS Omikuji Bonsai Finnish"
+language="fi"
+backend="omikuji"
+analyzer="voikko(fi)"
+vocab="kauno"
+cluster_balanced=false
+cluster_k=100
+max_depth=3
+min_df=2
+ngram=2
+transform="limit(5000)"
+access="hidden"
+
+[kauno-finna-bonsai-fi]
+name="KAUNO Finna Omikuji Bonsai Finnish"
+language="fi"
+backend="omikuji"
+analyzer="voikko(fi)"
+vocab="kauno"
+cluster_balanced=false
+cluster_k=100
+max_depth=3
+min_df=2
+ngram=2
+transform="limit(5000)"
+access="hidden"
+
+[kauno-fi]
+name="KAUNO suomi"
+language="fi"
+backend="nn_ensemble"
+sources="kauno-mllm-fi:0.1293,kauno-ks-bonsai-fi:0.5518,kauno-finna-bonsai-fi:0.3189"
+vocab="kauno"

reports/kauno-fi.json

@@ -0,0 +1,5 @@
+{
+  "F1@5": 0.4662634707739284,
+  "NDCG": 0.6133524179458618,
+  "Documents_evaluated": 1367
+}

reports/kauno-finna-bonsai-fi.json

@@ -0,0 +1,5 @@
+{
+  "F1@5": 0.2873918358957718,
+  "NDCG": 0.38556820154190063,
+  "Documents_evaluated": 1367
+}

reports/kauno-ks-bonsai-fi.json

@@ -0,0 +1,5 @@
+{
+  "F1@5": 0.39796013215991816,
+  "NDCG": 0.5301554799079895,
+  "Documents_evaluated": 1367
+}

reports/kauno-mllm-fi.json

@@ -0,0 +1,5 @@
+{
+  "F1@5": 0.18675745352815562,
+  "NDCG": 0.24778875708580017,
+  "Documents_evaluated": 1367
+}

requirements.txt

@@ -0,0 +1 @@
+annif[fasttext,omikuji,nn,voikko]==1.0.*

sync-model-data-ocp.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Runs rsync to transfer model data from the current directory to an OpenShift volume
+# that is attached to a pod which is running Annif. The instance
+# {api-annif-org,ai-finto-fi, etc.} to transfer to is given as the argument.
+# You need to be logged to the cluster with the oc tool.
+
+set -e
+
+if [ $# -ne 1 ]
+  then
+    echo "Not enough arguments; argument 1: destination_instance"
+    exit 1
+fi
+
+pod=$(oc get pods -l app.kubernetes.io/instance=$1,app.kubernetes.io/name=annif -o name)
+
+if [[ $pod = *[[:space:]]* ]]
+  then
+    echo "Multiple pod exists; using first"
+    pod=(${pod//$'\n'/ })
+fi
+echo "Target is "$pod
+pod=${pod#pod/}
+if [ -z "${pod}" ]
+  then
+    echo "No target pod found"
+    exit 1
+fi
+
+rsync --rsh='oc rsh' -avrL --exclude="*train*" --exclude="*zip" --inplace projects.d $pod:/annif-projects
+rsync --rsh='oc rsh' -avrL --exclude="*train*" --exclude="*zip" --inplace data/{projects,vocabs} $pod:/annif-projects/data