datasets update

README.md

@@ -12,107 +12,12 @@ datasets:
 - bigcode/starcoderdata
 # - Stackexchange
 # - CommonCrawl
-# - open-web-math/open-web-math # Phase 1
-# - math-ai/StackMathQA # Phase 2
+- open-web-math/open-web-math
+- math-ai/StackMathQA
 # - Arxiv
 # - Wikipedia
 # - conceptofmind/FLAN_2022 # Original link is broken, we used IBM's filtered version | Phase 2
-# - bigcode/commitpackft # Phase 2
-# - bigcode/oasst-octopack # Phase 2
-
-# Phase 1 datasets 
-- togethercomputer/RedPajama-Data-V2 # Common Crawl - CC (Redpajama v2)
-- togethercomputer/RedPajama-Data-1T # Books (Redpajama v1)
-- allenai/peS2o 
-- open-web-math/open-web-math 
-- EleutherAI/proof-pile-2 # Algebraic-stack (HF) 
-# - Code pile v2 w/o GPL (dp08)
-# - Webhose (dp08)
-# - Patents (dp08) 
-# - Arxiv (dp08)
-# - IEEE (dp08)
-# - DMMath (dp08)
-# - Financial research paper (dp08)
-# - Paper with code (dp08)
-# - Wikipedia (dp08)
-# - Stackexchange (dp08)
-# - doabooks (dp08)
-# - Freelaw (dp08)
-# - Pubmed (dp08)
-# - EDGAR (dp08)
-# - Secfiling (dp08)
-# - FIDC (dp08)
-# - Earning call transcript (dp08)
-# 
-# Phase 2 datasets: add high quality + instruction tuning datasets into the mixture
-# Hiqh quality: 
-# - sap_revised
-# - cybersecurity
-# - ibm-redbooks
-# - ibm.com
-# - superknowa
-# - multilingual – wikipedia + doabooks (de/es/fr/ja/pt/ar/cs/it/ko/nl/zh)
-# Instruction-tuning
 - nvidia/HelpSteer
-- garage-bAInd/Open-Platypus
-- mosaicml/dolly_hhrlhf
-- mosaicml/instruct-v3
-- conceptofmind/FLAN_2022
-- KnutJaegersberg/longinstruct
-- bigcode/oasst-octopack
-- CohereForAI/xP3x
-- math-ai/StackMathQA
-- math-ai/TemplateGSM
-- bugdaryan/sql-create-context-instruction
-- glaiveai/glaive-function-calling-v2
-- glaiveai/glaive-code-assistant-v3
-- cognitivecomputations/dolphin-coder
-- glaiveai/glaive-code-assistant
-- TokenBender/code_instructions_122k_alpaca_style
-- TIGER-Lab/MathInstruct
-- meta-math/MetaMathQA
-- tiedong/goat
-- CohereForAI/xP3x
-- bigcode/commitpack
-- bigcode/commitpackft
-- HuggingFaceTB/cosmopedia
-- deepmind/code_contests
-- ise-uiuc/Magicoder-Evol-Instruct-110K
-- ise-uiuc/Magicoder-OSS-Instruct-75K
-- theblackcat102/evol-codealpaca-v1
-- ajibawa-2023/Code-290k-ShareGPT
-- Locutusque/UltraTextbooks-2.0
-- teknium/OpenHermes-2.5
-- stingning/ultrachat
-# - API Blend
-# 
-# DATASET LINKS
-# NL
-# - nvidia/HelpSteer
-# - garage-bAInd/Open-Platypus
-# - mosaicml/dolly_hhrlhf
-# - mosaicml/instruct-v3
-# - conceptofmind/FLAN_2022
-# - KnutJaegersberg/longinstruct
-# - CohereForAI/xP3x
-# - HuggingFaceTB/cosmopedia
-# - open-web-math/open-web-math
-# - EleutherAI/proof-pile-2
-# - math-ai/StackMathQA
-# - math-ai/TemplateGSM
-# - IBM ConvAI 0111
-# - IBM Forca 30K
-# - IBM Hardcoded
-# Code
-# - bugdaryan/sql-create-context-instruction
-# - glaiveai/glaive-function-calling-v2
-# - cognitivecomputations/dolphin-coder
-# - glaiveai/glaive-code-
-# - bigcode/commitpackft
-# - TIGER-Lab/MathInstruct
-# - meta-math/MetaMathQA
-# - tiedong/goat
-# - CohereForAI/xP3x
 metrics:
 - code_eval
 library_name: transformers