"""Backfill course-outline checkpoints for COURSE/VIDEO resources that have none. Uses the lightweight importer (Coursera/DataCamp schema.org JSON-LD, YouTube chapter markers). Dry-run by default. `--apply` writes rows (create-if-absent, never clobbering existing checkpoints). `--emit-yaml` prints a `checkpoints:` snippet per resource for pasting into seed_data/resources.yaml (we never rewrite that curated file in place — PyYAML would strip its comments/ordering). Unextractable resources (Udemy, edX, SPA pages, single videos without chapter markers) are printed as a manual-authoring worklist; pair this with `manage.py check_checkpoints` to confirm the >=2 guarantee is met. """ import yaml from django.core.management.base import BaseCommand from django.db import transaction from django.db.models import Count from django.utils import timezone from apps.resources.importers import extract_checkpoints from apps.resources.models import Resource, ResourceCheckpoint SCOPED_TYPES = ("COURSE", "VIDEO") class Command(BaseCommand): help = ( "Draft checkpoints for COURSE/VIDEO resources with none via the course-" "outline importer. Dry-run unless --apply. --emit-yaml prints snippets " "for the seed file. Lists unextractable resources as a manual worklist." ) def add_arguments(self, parser): parser.add_argument( "--apply", action="store_true", help="Create ResourceCheckpoint rows (create-if-absent). " "Default: dry-run report only.", ) parser.add_argument( "--emit-yaml", action="store_true", help="Print a checkpoints: YAML snippet per resource (for manual " "paste into seed_data/resources.yaml).", ) parser.add_argument( "--type", choices=list(SCOPED_TYPES), default=None, help="Restrict to one type (default: both COURSE and VIDEO).", ) parser.add_argument( "--limit", type=int, default=None, help="Process at most N resources (for spot checks).", ) parser.add_argument( "--replace", action="store_true", help="Refresh resources that ALREADY have checkpoints: when " "extraction yields >=2, delete and recreate from the import " "(e.g. real YouTube chapters once YOUTUBE_API_KEY is set). " "Never blanks a resource (extraction <2 leaves rows intact). " "Requires --apply. NOTE: deleting checkpoints cascades " "UserCheckpointProgress.", ) def handle(self, *args, **options): replace = options["replace"] if replace and not options["apply"]: self.stderr.write(self.style.ERROR( "--replace requires --apply (it deletes existing rows)." )) return types = [options["type"]] if options["type"] else list(SCOPED_TYPES) qs = Resource.objects.filter(type__in=types).annotate(n=Count("checkpoints")) # Default backfill targets resources with NO checkpoints. --replace # instead targets the ones that DO have them (to refresh in place). qs = qs.filter(n__gte=1) if replace else qs.filter(n=0) qs = qs.order_by("type", "title") if options["limit"]: qs = qs[: options["limit"]] extracted = created_rows = 0 failed: list[tuple[Resource, str]] = [] for resource in qs: result = extract_checkpoints(resource.url) if not result.checkpoints: failed.append((resource, result.note)) self.stdout.write( f" [--] {resource.title} ({resource.provider}) - {result.note}" ) continue extracted += 1 self.stdout.write(self.style.SUCCESS( f" [ok] {resource.title} ({resource.provider}) - " f"{len(result.checkpoints)} via {result.source}" )) for c in result.checkpoints: self.stdout.write(f" {c.order_index}. {c.title}") if options["emit_yaml"]: self._emit_yaml(resource, result) if not options["apply"]: continue rows = [ ResourceCheckpoint( resource=resource, order_index=c.order_index, title=c.title[:500], url_fragment=c.url_fragment, estimated_minutes=c.estimated_minutes, source=result.source, extracted_at=timezone.now(), ) for c in result.checkpoints ] if replace: # Refresh in place — but never blank a resource: only swap when # the import gives a real list (>=2). Delete + recreate together # so a failure can't leave the resource at 0 checkpoints. if len(rows) < 2: self.stdout.write( f" (kept existing — extraction returned " f"{len(rows)} < 2)" ) continue with transaction.atomic(): resource.checkpoints.all().delete() ResourceCheckpoint.objects.bulk_create(rows) created_rows += len(rows) self.stdout.write(f" (replaced with {len(rows)} rows)") elif not resource.checkpoints.exists(): ResourceCheckpoint.objects.bulk_create(rows) created_rows += len(rows) if not options["apply"]: mode = "DRY-RUN (pass --apply to persist)" elif replace: mode = "REPLACED" else: mode = "APPLIED" self.stdout.write("") self.stdout.write(self.style.MIGRATE_HEADING(f"== {mode} ==")) verb = "rows replaced" if replace else "rows created" line = f"Extractable: {extracted} | needs manual authoring: {len(failed)}" if options["apply"]: line += f" | {verb}: {created_rows}" self.stdout.write(line) if failed: self.stdout.write(self.style.WARNING( "Manual-authoring worklist (COURSE/VIDEO with no auto-outline):" )) for resource, _ in failed: self.stdout.write( f" - [{resource.type}] {resource.title} ({resource.provider})" ) def _emit_yaml(self, resource, result): snippet = { "checkpoints": [c.title for c in result.checkpoints], "checkpoint_source": result.source, } self.stdout.write(f"# {resource.title} <{resource.url}>") self.stdout.write(yaml.safe_dump( snippet, allow_unicode=True, sort_keys=False, default_flow_style=False ))