Spaces:
Runtime error
Runtime error
| """Backfill course-outline checkpoints for COURSE/VIDEO resources that have none. | |
| Uses the lightweight importer (Coursera/DataCamp schema.org JSON-LD, YouTube | |
| chapter markers). Dry-run by default. `--apply` writes rows (create-if-absent, | |
| never clobbering existing checkpoints). `--emit-yaml` prints a `checkpoints:` | |
| snippet per resource for pasting into seed_data/resources.yaml (we never rewrite | |
| that curated file in place — PyYAML would strip its comments/ordering). | |
| Unextractable resources (Udemy, edX, SPA pages, single videos without chapter | |
| markers) are printed as a manual-authoring worklist; pair this with | |
| `manage.py check_checkpoints` to confirm the >=2 guarantee is met. | |
| """ | |
| import yaml | |
| from django.core.management.base import BaseCommand | |
| from django.db import transaction | |
| from django.db.models import Count | |
| from django.utils import timezone | |
| from apps.resources.importers import extract_checkpoints | |
| from apps.resources.models import Resource, ResourceCheckpoint | |
| SCOPED_TYPES = ("COURSE", "VIDEO") | |
| class Command(BaseCommand): | |
| help = ( | |
| "Draft checkpoints for COURSE/VIDEO resources with none via the course-" | |
| "outline importer. Dry-run unless --apply. --emit-yaml prints snippets " | |
| "for the seed file. Lists unextractable resources as a manual worklist." | |
| ) | |
| def add_arguments(self, parser): | |
| parser.add_argument( | |
| "--apply", action="store_true", | |
| help="Create ResourceCheckpoint rows (create-if-absent). " | |
| "Default: dry-run report only.", | |
| ) | |
| parser.add_argument( | |
| "--emit-yaml", action="store_true", | |
| help="Print a checkpoints: YAML snippet per resource (for manual " | |
| "paste into seed_data/resources.yaml).", | |
| ) | |
| parser.add_argument( | |
| "--type", choices=list(SCOPED_TYPES), default=None, | |
| help="Restrict to one type (default: both COURSE and VIDEO).", | |
| ) | |
| parser.add_argument( | |
| "--limit", type=int, default=None, | |
| help="Process at most N resources (for spot checks).", | |
| ) | |
| parser.add_argument( | |
| "--replace", action="store_true", | |
| help="Refresh resources that ALREADY have checkpoints: when " | |
| "extraction yields >=2, delete and recreate from the import " | |
| "(e.g. real YouTube chapters once YOUTUBE_API_KEY is set). " | |
| "Never blanks a resource (extraction <2 leaves rows intact). " | |
| "Requires --apply. NOTE: deleting checkpoints cascades " | |
| "UserCheckpointProgress.", | |
| ) | |
| def handle(self, *args, **options): | |
| replace = options["replace"] | |
| if replace and not options["apply"]: | |
| self.stderr.write(self.style.ERROR( | |
| "--replace requires --apply (it deletes existing rows)." | |
| )) | |
| return | |
| types = [options["type"]] if options["type"] else list(SCOPED_TYPES) | |
| qs = Resource.objects.filter(type__in=types).annotate(n=Count("checkpoints")) | |
| # Default backfill targets resources with NO checkpoints. --replace | |
| # instead targets the ones that DO have them (to refresh in place). | |
| qs = qs.filter(n__gte=1) if replace else qs.filter(n=0) | |
| qs = qs.order_by("type", "title") | |
| if options["limit"]: | |
| qs = qs[: options["limit"]] | |
| extracted = created_rows = 0 | |
| failed: list[tuple[Resource, str]] = [] | |
| for resource in qs: | |
| result = extract_checkpoints(resource.url) | |
| if not result.checkpoints: | |
| failed.append((resource, result.note)) | |
| self.stdout.write( | |
| f" [--] {resource.title} ({resource.provider}) - {result.note}" | |
| ) | |
| continue | |
| extracted += 1 | |
| self.stdout.write(self.style.SUCCESS( | |
| f" [ok] {resource.title} ({resource.provider}) - " | |
| f"{len(result.checkpoints)} via {result.source}" | |
| )) | |
| for c in result.checkpoints: | |
| self.stdout.write(f" {c.order_index}. {c.title}") | |
| if options["emit_yaml"]: | |
| self._emit_yaml(resource, result) | |
| if not options["apply"]: | |
| continue | |
| rows = [ | |
| ResourceCheckpoint( | |
| resource=resource, | |
| order_index=c.order_index, | |
| title=c.title[:500], | |
| url_fragment=c.url_fragment, | |
| estimated_minutes=c.estimated_minutes, | |
| source=result.source, | |
| extracted_at=timezone.now(), | |
| ) | |
| for c in result.checkpoints | |
| ] | |
| if replace: | |
| # Refresh in place — but never blank a resource: only swap when | |
| # the import gives a real list (>=2). Delete + recreate together | |
| # so a failure can't leave the resource at 0 checkpoints. | |
| if len(rows) < 2: | |
| self.stdout.write( | |
| f" (kept existing — extraction returned " | |
| f"{len(rows)} < 2)" | |
| ) | |
| continue | |
| with transaction.atomic(): | |
| resource.checkpoints.all().delete() | |
| ResourceCheckpoint.objects.bulk_create(rows) | |
| created_rows += len(rows) | |
| self.stdout.write(f" (replaced with {len(rows)} rows)") | |
| elif not resource.checkpoints.exists(): | |
| ResourceCheckpoint.objects.bulk_create(rows) | |
| created_rows += len(rows) | |
| if not options["apply"]: | |
| mode = "DRY-RUN (pass --apply to persist)" | |
| elif replace: | |
| mode = "REPLACED" | |
| else: | |
| mode = "APPLIED" | |
| self.stdout.write("") | |
| self.stdout.write(self.style.MIGRATE_HEADING(f"== {mode} ==")) | |
| verb = "rows replaced" if replace else "rows created" | |
| line = f"Extractable: {extracted} | needs manual authoring: {len(failed)}" | |
| if options["apply"]: | |
| line += f" | {verb}: {created_rows}" | |
| self.stdout.write(line) | |
| if failed: | |
| self.stdout.write(self.style.WARNING( | |
| "Manual-authoring worklist (COURSE/VIDEO with no auto-outline):" | |
| )) | |
| for resource, _ in failed: | |
| self.stdout.write( | |
| f" - [{resource.type}] {resource.title} ({resource.provider})" | |
| ) | |
| def _emit_yaml(self, resource, result): | |
| snippet = { | |
| "checkpoints": [c.title for c in result.checkpoints], | |
| "checkpoint_source": result.source, | |
| } | |
| self.stdout.write(f"# {resource.title} <{resource.url}>") | |
| self.stdout.write(yaml.safe_dump( | |
| snippet, allow_unicode=True, sort_keys=False, default_flow_style=False | |
| )) | |