{ "cells": [ { "cell_type": "code", "metadata": { "collapsed": false, "scrolled": true }, "source": [ "%uv pip install datasets huggingface_hub" ], "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\u001b[2mUsing Python 3.12.6 environment at: /usr/local\u001b[0m\r\n", "\u001b[37m\u280b\u001b[0m \u001b[2mResolving dependencies... \u001b[0m\r\u001b[2K\u001b[37m\u280b\u001b[0m \u001b[2mResolving dependencies... \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mResolving dependencies... \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mdatasets==5.0.0 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhuggingface-hub==0.34.4 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mfilelock==3.13.1 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mnumpy==2.1.2 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpyarrow==25.0.0 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mdill==0.4.1 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpandas==2.3.2 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mnumpy==2.1.2 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mrequests==2.32.5 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhttpx==0.28.1 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mtqdm==4.67.1 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mxxhash==3.8.1 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mmultiprocess==0.70.19 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mfsspec==2024.6.1 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mfsspec==2024.6.1 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpackaging==25.0 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mpyyaml==6.0.2 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mtyping-extensions==4.12.2 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhf-xet==1.1.9 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mhf-xet==1.1.9 \u001b[0m\r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mfrozenlist==1.4.1 \u001b[0m\r\u001b[2K\u001b[2mResolved \u001b[1m36 packages\u001b[0m \u001b[2min 159ms\u001b[0m\u001b[0m\r\n", "\u001b[37m\u280b\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/0) \r\u001b[2K\u001b[37m\u280b\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5) \r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5) \r\u001b[2K\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m 0 B/215.30 KiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 16.00 KiB/215.30 KiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 16.00 KiB/215.30 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m 0 B/47.77 MiB \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 16.00 KiB/215.30 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m 0 B/117.21 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 16.00 KiB/215.30 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 16.00 KiB/215.30 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 32.00 KiB/215.30 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m-------\u001b[2m-----------------------\u001b[0m\u001b[0m 48.00 KiB/215.30 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m 0 B/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.84 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m\u001b[2m------------------------------\u001b[0m\u001b[0m 0 B/146.76 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.84 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.83 KiB/146.76 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.84 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.80 KiB/47.77 MiB \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.83 KiB/146.76 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.84 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 30.80 KiB/47.77 MiB \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.83 KiB/146.76 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.84 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 46.80 KiB/47.77 MiB \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.83 KiB/146.76 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.84 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 48.06 KiB/47.77 MiB \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m--------\u001b[2m----------------------\u001b[0m\u001b[0m 30.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m----\u001b[2m--------------------------\u001b[0m\u001b[0m 14.83 KiB/146.76 KiB\r\n", "\u001b[2mxxhash \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 62.84 KiB/215.30 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 14.84 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 48.06 KiB/47.77 MiB \u001b[5A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[5A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m---------------------\u001b[2m---------\u001b[0m\u001b[0m 78.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m-------------\u001b[2m-----------------\u001b[0m\u001b[0m 62.83 KiB/146.76 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m---------\u001b[2m---------------------\u001b[0m\u001b[0m 158.74 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 700.38 KiB/47.77 MiB \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m---------------------\u001b[2m---------\u001b[0m\u001b[0m 78.81 KiB/117.21 KiB\r\n", "\u001b[2mmultiprocess\u001b[0m \u001b[32m-------------\u001b[2m-----------------\u001b[0m\u001b[0m 62.83 KiB/146.76 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m-------------\u001b[2m-----------------\u001b[0m\u001b[0m 222.74 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-\u001b[2m-----------------------------\u001b[0m\u001b[0m 1.07 MiB/47.77 MiB \u001b[4A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[4A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdill \u001b[0m \u001b[32m------------------------------\u001b[2m\u001b[0m\u001b[0m 117.21 KiB/117.21 KiB\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m------------------\u001b[2m------------\u001b[0m\u001b[0m 318.74 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 2.57 MiB/47.77 MiB \u001b[3A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[3A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m------------------\u001b[2m------------\u001b[0m\u001b[0m 318.74 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m--\u001b[2m----------------------------\u001b[0m\u001b[0m 2.59 MiB/47.77 MiB \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m--------------------\u001b[2m----------\u001b[0m\u001b[0m 350.74 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m---\u001b[2m---------------------------\u001b[0m\u001b[0m 3.64 MiB/47.77 MiB \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2819\u001b[0m \u001b[2mPreparing packages...\u001b[0m (0/5)\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m----------------------\u001b[2m--------\u001b[0m\u001b[0m 382.74 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-----\u001b[2m-------------------------\u001b[0m\u001b[0m 7.28 MiB/47.77 MiB \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n", "\u001b[2mdatasets \u001b[0m \u001b[32m------------------------------\u001b[2m\u001b[0m\u001b[0m 526.74 KiB/542.07 KiB\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-------\u001b[2m-----------------------\u001b[0m\u001b[0m 10.50 MiB/47.77 MiB \u001b[2A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[2A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m--------\u001b[2m----------------------\u001b[0m\u001b[0m 12.24 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m----------\u001b[2m--------------------\u001b[0m\u001b[0m 14.62 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m------------\u001b[2m------------------\u001b[0m\u001b[0m 18.60 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m--------------\u001b[2m----------------\u001b[0m\u001b[0m 22.19 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2839\u001b[0m \u001b[2mPreparing packages...\u001b[0m (3/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m----------------\u001b[2m--------------\u001b[0m\u001b[0m 25.34 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-------------------\u001b[2m-----------\u001b[0m\u001b[0m 29.08 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m---------------------\u001b[2m---------\u001b[0m\u001b[0m 33.08 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m------------------------\u001b[2m------\u001b[0m\u001b[0m 37.15 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u2838\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m--------------------------\u001b[2m----\u001b[0m\u001b[0m 39.96 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u283c\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m----------------------------\u001b[2m--\u001b[0m\u001b[0m 43.75 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u283c\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m-----------------------------\u001b[2m-\u001b[0m\u001b[0m 45.97 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u283c\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5)\r\n", "\u001b[2mpyarrow \u001b[0m \u001b[32m------------------------------\u001b[2m\u001b[0m\u001b[0m 46.76 MiB/47.77 MiB \u001b[1A\r\u001b[2K\u001b[1B\r\u001b[2K\u001b[1A\u001b[37m\u283c\u001b[0m \u001b[2mPreparing packages...\u001b[0m (4/5) \r\u001b[2K\u001b[2mPrepared \u001b[1m5 packages\u001b[0m \u001b[2min 793ms\u001b[0m\u001b[0m\r\n", "\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [0/0] \u001b[2mInstalling wheels... \u001b[0m\r\u001b[2K\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [0/5] \u001b[2mInstalling wheels... \u001b[0m\r\u001b[2K\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [0/5] \u001b[2mxxhash==3.8.1 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [1/5] \u001b[2mxxhash==3.8.1 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [1/5] \u001b[2mmultiprocess==0.70.19 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [2/5] \u001b[2mmultiprocess==0.70.19 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [2/5] \u001b[2mdill==0.4.1 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [3/5] \u001b[2mdill==0.4.1 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591\u2591\u2591\u2591\u2591 [3/5] \u001b[2mdatasets==5.0.0 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591 [4/5] \u001b[2mdatasets==5.0.0 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2591\u2591\u2591\u2591 [4/5] \u001b[2mpyarrow==25.0.0 \u001b[0m\r\u001b[2K\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 [5/5] \u001b[2mpyarrow==25.0.0 \u001b[0m\r\u001b[2K\u001b[2mInstalled \u001b[1m5 packages\u001b[0m \u001b[2min 43ms\u001b[0m\u001b[0m\r\n", " \u001b[32m+\u001b[39m \u001b[1mdatasets\u001b[0m\u001b[2m==5.0.0\u001b[0m\r\n", " \u001b[32m+\u001b[39m \u001b[1mdill\u001b[0m\u001b[2m==0.4.1\u001b[0m\r\n", " \u001b[32m+\u001b[39m \u001b[1mmultiprocess\u001b[0m\u001b[2m==0.70.19\u001b[0m\r\n", " \u001b[32m+\u001b[39m \u001b[1mpyarrow\u001b[0m\u001b[2m==25.0.0\u001b[0m\r\n", " \u001b[32m+\u001b[39m \u001b[1mxxhash\u001b[0m\u001b[2m==3.8.1\u001b[0m\r\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ] }, { "cell_type": "code", "metadata": { "collapsed": false, "scrolled": true }, "source": [ "import torch\n", "import random\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "from torch.utils.data import DataLoader\n", "from datasets import load_dataset\n", "from huggingface_hub import PyTorchModelHubMixin, notebook_login" ], "execution_count": 7, "outputs": [] }, { "cell_type": "code", "metadata": { "collapsed": false, "scrolled": true }, "source": [ "ds_splits = load_dataset(\"jakartaresearch/cerpen-corpus\", split=\"train\", revision=\"refs/convert/parquet\").train_test_split(test_size=0.1)\n", "ds_train = ds_splits[\"train\"]\n", "ds_test = ds_splits[\"test\"]\n", "\n", "def gen(rows):\n", " sentences = []\n", " for content in rows[\"content\"]:\n", " stripped = ''.join(c.lower() for c in content if c.isalpha() or c == ' ')\n", " sentences.append(stripped)\n", " return dict(sentence1=sentences)\n", "\n", "ds_train = ds_train.map(gen, batched=True, remove_columns=ds_train.column_names)\n", "ds_test = ds_test.map(gen, batched=True, remove_columns=ds_test.column_names)" ], "execution_count": 8, "outputs": [ { "output_type": "display_data", "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "044dc56786ea411381769c8664d30915", "version_minor": 0.0, "version_major": 2.0 }, "text/plain": "0000.parquet: 0%| | 0.00/209k [00:00', embedding)" ], "execution_count": 17, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "ada -> tensor([[-0.1620, 2.3667, -1.6400, 1.0723, 1.3573, 1.7939, -1.9762, -0.7847,\n", " -0.6792, 1.1668, 4.2722, -0.0947, 0.8006, -0.6954, 0.0548, 0.3087,\n", " 0.7016, -0.3470, 0.2811, 0.5155, 2.1355, 0.3513, -0.6381, 1.2816,\n", " -0.7497, -1.1857, -1.7049, -0.3956, 1.2827, -2.2924, -2.2530, -2.2833,\n", " -1.9617, 0.1064, 2.3472, 0.0685, -0.1007, 0.6174, 1.8690, 0.5491,\n", " 0.3912, -2.4253, -0.3393, 2.0718, 0.2526, 1.3298, 0.8708, 2.1571,\n", " -0.8963, -1.6683, 0.8791, -1.1831, 1.0959, -0.8845, -1.7375, 0.0917,\n", " 0.3306, 0.5309, 0.0951, 2.8126, 0.3754, 2.0260, -0.8577, 2.0378,\n", " 1.4960, 2.0437, -1.2764, 2.3857, 3.0320, 0.1518, -0.1080, -0.1423,\n", " -1.1395, -0.1090, 2.5674, 1.9636, -1.0849, 0.4864, 1.4419, 0.3830,\n", " -0.9491, -3.2826, 1.1652, 1.0227, 0.9529, 2.0130, 1.3376, 1.0838,\n", " -1.5363, 1.0757, -2.3646, 0.7869, 4.1987, -0.5543, 2.1535, -2.2712,\n", " 3.0948, -0.6080, 1.2531, -0.1201]], device='cuda:0',\n", " grad_fn=)\n" ] } ] }, { "cell_type": "code", "metadata": { "collapsed": false, "scrolled": true }, "source": [ "PurpleCBoW" ], "execution_count": 19, "outputs": [ { "output_type": "execute_result", "execution_count": 19, "data": { "text/plain": "CBOW(\n (embeddings): Embedding(8254, 100)\n (linear): Linear(in_features=100, out_features=8254, bias=True)\n)" }, "metadata": {} } ] }, { "cell_type": "code", "metadata": { "collapsed": false, "scrolled": true }, "source": [ "notebook_login()" ], "execution_count": 22, "outputs": [ { "output_type": "display_data", "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "630df8641a9c4094b70be193cd4ebfbb", "version_minor": 0.0, "version_major": 2.0 }, "text/plain": "VBox(children=(HTML(value='