Upload folder using huggingface_hub

.python_tmp/05ef2759-9606-4dbe-8aab-d60dc9b9acec.py

@@ -0,0 +1,61 @@
+from huggingface_hub import HfApi, hf_hub_download
+import os, json
+root = os.getcwd()
+api = HfApi()
+with open('hf_token.txt') as f:
+    token = f.read().strip()
+who = api.whoami(token=token)
+username = who.get('name') or who.get('login')
+repo_id = f"{username}/MyAwesomeModel-CleanupRelease"
+
+# canonical bytes from local step_100
+canon_path = os.path.join(root,'checkpoints','step_100','config.json')
+with open(canon_path,'rb') as f:
+    canonical = f.read()
+
+# gather local config paths
+local_configs = []
+for dirpath, dirnames, filenames in os.walk(os.path.join(root,'checkpoints')):
+    for fn in filenames:
+        if fn=='config.json':
+            local_configs.append(os.path.join(dirpath,fn))
+local_configs.sort()
+
+mismatches = []
+for p in local_configs:
+    rel = os.path.relpath(p, root)
+    try:
+        dl = hf_hub_download(repo_id=repo_id, filename=rel, token=token, repo_type='model')
+    except Exception as e:
+        mismatches.append((rel, 'download_error', str(e)))
+        continue
+    with open(dl,'rb') as f:
+        b = f.read()
+    if b != canonical:
+        mismatches.append((rel, 'byte_mismatch', len(b)))
+
+# check repo .swp
+files = api.list_repo_files(repo_id=repo_id, token=token)
+swp_files = [f for f in files if f.endswith('.swp')]
+
+# check local swap removed
+swap_local = os.path.join(root,'evaluation','.setup.py.swp')
+local_swap_exists = os.path.exists(swap_local)
+
+print('local_swap_exists', local_swap_exists)
+print('repo_swp_files', swp_files)
+print('mismatches', mismatches)
+
+# show canonical formatting details
+s = canonical.decode('utf-8')
+lines = s.splitlines()
+print('Canonical raw repr:', repr(canonical))
+print('Canonical text:\n', s)
+print('Line count', len(lines))
+# determine indentation by looking at second line leading spaces
+if len(lines)>=2:
+    import re
+    m = re.match(r"^(\s*)", lines[1])
+    print('Indentation chars on line 2:', repr(m.group(1)))
+# trailing newline?
+print('Ends with newline?', canonical.endswith(b'\n'))

.python_tmp/28a4b2a5-22d6-4929-a9c6-bb8ee74c9abe.py

@@ -0,0 +1,44 @@
+from huggingface_hub import HfApi, upload_folder, hf_hub_download
+import os
+root = os.getcwd()
+with open('hf_token.txt') as f:
+    token = f.read().strip()
+api = HfApi()
+who = api.whoami(token=token)
+username = who.get('name') or who.get('login')
+repo_name = 'MyAwesomeModel-CleanupRelease'
+repo_id = f"{username}/{repo_name}"
+# create repo with correct signature
+try:
+    api.create_repo(repo_id=repo_id, token=token, exist_ok=True)
+    print('create_repo OK')
+except Exception as e:
+    print('create_repo error', e)
+
+# upload using upload_folder signature
+try:
+    upload_folder(repo_id=repo_id, folder_path=root, path_in_repo='.', token=token, repo_type='model', ignore_patterns=['*.swp'])
+    print('upload_folder OK')
+except Exception as e:
+    print('upload_folder error', e)
+
+# list files
+try:
+    files = api.list_repo_files(repo_id=repo_id, token=token)
+    print('files count', len(files))
+    swp_files = [f for f in files if f.endswith('.swp')]
+    print('.swp files in repo:', swp_files)
+except Exception as e:
+    print('list_repo_files error', e)
+
+# download one config to verify
+rel = os.path.relpath(os.path.join(root,'checkpoints','step_100','config.json'), root)
+print('rel', rel)
+try:
+    dl = hf_hub_download(repo_id=repo_id, filename=rel, token=token, repo_type='model')
+    print('downloaded to', dl)
+    with open(dl,'rb') as f:
+        b = f.read()
+    print('size', len(b))
+except Exception as e:
+    print('download error', e)

.python_tmp/40c80db1-8de5-4cb2-8f64-de819548ed08.py

@@ -0,0 +1,142 @@
+import os, sys, json
+from collections import Counter
+
+root = os.getcwd()
+# find config files
+config_paths = []
+for dirpath, dirnames, filenames in os.walk(os.path.join(root, 'checkpoints')):
+    for fn in filenames:
+        if fn == 'config.json':
+            config_paths.append(os.path.join(dirpath, fn))
+config_paths.sort()
+print('Found', len(config_paths), 'config.json files')
+
+# read bytes
+bytes_map = {}
+for p in config_paths:
+    with open(p, 'rb') as f:
+        b = f.read()
+    bytes_map[p] = b
+
+# determine most common bytes
+counter = Counter(bytes_map.values())
+most_common = counter.most_common(1)
+if not most_common:
+    print('No config files found, exiting')
+    sys.exit(1)
+canonical_bytes, count = most_common[0]
+print('Canonical bytes occur', count, 'times out of', len(config_paths))
+
+# show canonical as repr
+print('Canonical bytes repr:\n', canonical_bytes)
+
+# rewrite outliers after validating JSON content identical
+rewritten = []
+mismatched_content = []
+for p, b in bytes_map.items():
+    if b != canonical_bytes:
+        # validate JSON content equality
+        try:
+            js_current = json.loads(b.decode('utf-8'))
+            js_canonical = json.loads(canonical_bytes.decode('utf-8'))
+        except Exception as e:
+            print('Failed to parse JSON for', p, 'err', e)
+            mismatched_content.append(p)
+            continue
+        if js_current != js_canonical:
+            print('JSON content differs for', p)
+            mismatched_content.append(p)
+            continue
+        # write canonical bytes
+        with open(p, 'wb') as f:
+            f.write(canonical_bytes)
+        rewritten.append(p)
+
+print('Rewrote', len(rewritten), 'files')
+if mismatched_content:
+    print('Files with mismatched JSON content:', mismatched_content)
+
+# remove swap file if exists
+swap = os.path.join(root, 'evaluation', '.setup.py.swp')
+if os.path.exists(swap):
+    os.remove(swap)
+    print('Removed swap file', swap)
+else:
+    print('No swap file found at', swap)
+
+# Now upload to Hugging Face
+try:
+    from huggingface_hub import HfApi, upload_folder, hf_hub_download
+except Exception as e:
+    print('huggingface_hub not installed:', e)
+    raise
+
+# read token
+token_path = os.path.join(root, 'hf_token.txt')
+if not os.path.exists(token_path):
+    print('hf_token.txt not found')
+    sys.exit(1)
+with open(token_path, 'r') as f:
+    token = f.read().strip()
+api = HfApi()
+who = api.whoami(token=token)
+username = who.get('name') or who.get('login')
+if not username:
+    print('Failed to get username from token whoami:', who)
+    sys.exit(1)
+repo_name = 'MyAwesomeModel-CleanupRelease'
+repo_id = f"{username}/{repo_name}"
+print('Repo id:', repo_id)
+
+# create repo if needed
+try:
+    api.create_repo(name=repo_name, token=token, exist_ok=True)
+    print('Created or found repo')
+except Exception as e:
+    print('create_repo error (continuing if exists):', e)
+
+# upload folder
+print('Uploading folder... this may take a while')
+upload_folder(
+    repo_id=repo_id,
+    repo_type='model',
+    local_dir=root,
+    path_in_repo='.',
+    token=token,
+    ignore_patterns=['*.swp']
+)
+print('Upload finished')
+
+# verify
+files = api.list_repo_files(repo_id=repo_id, token=token)
+print('Repo files count:', len(files))
+# check .swp absence
+swp_present = any(f.endswith('.swp') for f in files)
+print('.swp present in repo files?', swp_present)
+
+# For each checkpoints config, download and compare bytes
+mismatches_repo = []
+for p in config_paths:
+    # compute repo path relative
+    rel = os.path.relpath(p, root)
+    # hf_hub_download expects repo_id and filename
+    try:
+        local_dl = hf_hub_download(repo_id=repo_id, filename=rel, token=token, repo_type='model')
+    except Exception as e:
+        print('Failed to download', rel, 'err', e)
+        mismatches_repo.append(rel)
+        continue
+    with open(local_dl, 'rb') as f:
+        b = f.read()
+    if b != canonical_bytes:
+        mismatches_repo.append(rel)
+
+print('Mismatched files in repo:', mismatches_repo)
+
+# Output summary
+print('SUMMARY')
+print('canonical_bytes_count=', count)
+print('rewritten=', rewritten)
+print('swap_removed=', not os.path.exists(swap))
+print('swp_present_in_repo=', swp_present)
+print('mismatched_in_repo=', mismatches_repo)

.python_tmp/54bed8a4-2e49-4458-983f-45b2d078a179.py

@@ -0,0 +1,51 @@
+from huggingface_hub import HfApi, upload_folder, hf_hub_download
+import os
+root = os.getcwd()
+with open('hf_token.txt') as f:
+    token = f.read().strip()
+api = HfApi()
+who = api.whoami(token=token)
+username = who.get('name') or who.get('login')
+repo_name = 'MyAwesomeModel-CleanupRelease'
+repo_id = f"{username}/{repo_name}"
+
+# ensure repo
+api.create_repo(repo_id=repo_id, token=token, exist_ok=True)
+print('Repo ready:', repo_id)
+
+# upload, excluding hf_token.txt and .swp
+try:
+    upload_folder(repo_id=repo_id, folder_path=root, path_in_repo='.', token=token, repo_type='model', ignore_patterns=['*.swp','hf_token.txt'])
+    print('Upload completed')
+except Exception as e:
+    print('Upload failed:', e)
+
+# list files
+files = api.list_repo_files(repo_id=repo_id, token=token)
+print('Repo files count:', len(files))
+swp_files = [f for f in files if f.endswith('.swp')]
+print('.swp files in repo:', swp_files)
+
+# load canonical bytes
+canon_path = os.path.join(root,'checkpoints','step_100','config.json')
+with open(canon_path,'rb') as f:
+    canonical = f.read()
+
+# verify each config
+mismatches = []
+for dirpath, dirnames, filenames in os.walk(os.path.join(root,'checkpoints')):
+    for fn in filenames:
+        if fn == 'config.json':
+            local_p = os.path.join(dirpath, fn)
+            rel = os.path.relpath(local_p, root)
+            try:
+                dl = hf_hub_download(repo_id=repo_id, filename=rel, token=token, repo_type='model')
+            except Exception as e:
+                mismatches.append((rel, 'download_error', str(e)))
+                continue
+            with open(dl,'rb') as f:
+                b = f.read()
+            if b != canonical:
+                mismatches.append((rel, 'byte_mismatch', len(b)))
+
+print('mismatches after upload:', mismatches)

.python_tmp/bf6a9154-6953-4e1f-85eb-c9a5fb22d8d2.py

@@ -0,0 +1,9 @@
+import inspect
+from huggingface_hub import upload_folder, HfApi
+print('upload_folder signature:')
+print(inspect.signature(upload_folder))
+print('\nHfApi.create_repo signature:')
+print(inspect.signature(HfApi.create_repo))
+print('\nHfApi.upload_folder exists?', hasattr(HfApi, 'upload_folder'))
+if hasattr(HfApi, 'upload_folder'):
+    print('HfApi.upload_folder signature:', inspect.signature(HfApi.upload_folder))

checkpoints/step_200/config.json

@@ -1,6 +1,4 @@
 {
     "model_type": "bert",
-    "architectures": [
-        "BertModel"
-    ]
-}
+    "architectures": ["BertModel"]
+  }

checkpoints/step_300/config.json

@@ -1,6 +1,4 @@
 {
     "model_type": "bert",
-    "architectures": [
-        "BertModel"
-    ]
-}
+    "architectures": ["BertModel"]
+  }