Update CoreML model weights and manifest identifiers, add panorama to 3D Gaussian splat conversion CLI tool, implement horizontal seam blending to fix depth artifacts, update license to Apache-2.0 and document workflow, add demonstration GIF and regenerate test outputs

.gitattributes

@@ -5,3 +5,6 @@
 *.npy filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
 *.mlpackage filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.ply filter=lfs diff=lfs merge=lfs -text

360splat.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59a53ccb5c01edf1b9515d98db27f27e773c2040c0c78e885ccc0494c4f39d1b
+size 7089875

DAPModel.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e408640d06b3e1aad41fa3fe25e13a584a488811560a5ac8561f0c871fdd4c15
-size 402181
+oid sha256:5696ae403e0a1861cc444ab08f34fbb5581c928ea244c2fca2a03fe395e0378e
+size 402224

DAPModel.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7c1c0140c43fddf3b54af7d7c457d36c17d4672b83a22eac3362994c43ceeea
-size 1334575872
+oid sha256:8117c7e3129387f5219184f5e2d3a5c7c946c6f7905d39737a895dc29d6177c9
+size 1344013056

DAPModel.mlpackage/Manifest.json

@@ -1,18 +1,18 @@
 {
     "fileFormatVersion": "1.0.0",
     "itemInfoEntries": {
-        "2D7AE079-9AC7-4221-BC61-F6AA0F4AE8E0": {
+        "47133C03-7196-4F41-A4BA-71E0277FD33B": {
             "author": "com.apple.CoreML",
             "description": "CoreML Model Specification",
             "name": "model.mlmodel",
             "path": "com.apple.CoreML/model.mlmodel"
         },
-        "ABE41932-B39B-482F-AFF2-D5F1AFD7CFB8": {
+        "BDE59133-8A31-4AEB-B879-19F2D41BD3A9": {
             "author": "com.apple.CoreML",
             "description": "CoreML Model Weights",
             "name": "weights",
             "path": "com.apple.CoreML/weights"
         }
     },
-    "rootModelIdentifier": "2D7AE079-9AC7-4221-BC61-F6AA0F4AE8E0"
+    "rootModelIdentifier": "47133C03-7196-4F41-A4BA-71E0277FD33B"
 }

PanoramaSplat.swift

@@ -0,0 +1,415 @@
+//
+//  PanoramaSplat.swift
+//  Convert equirectangular 360° panoramas to 3D Gaussian splat PLY files
+//
+//  Uses a DAP CoreML depth model to estimate per-pixel depth from an
+//  equirectangular panorama, then projects each pixel onto a sphere
+//  to produce one Gaussian per pixel.
+//
+//  Usage:
+//    swiftc -O -o panorama_splat PanoramaSplat.swift \
+//        -framework CoreML -framework Vision -framework CoreImage \
+//        -framework CoreGraphics -framework AppKit
+//    ./panorama_splat -m DAPModel.mlpackage -i panorama.jpg -o scene.ply -r 5.0
+
+import Foundation
+import CoreML
+import Vision
+import CoreImage
+import CoreGraphics
+import AppKit
+
+// MARK: - Command Line Arguments
+
+struct CLIArgs {
+    let modelPath: URL
+    let imagePath: URL
+    let outputPath: URL
+    let radius: Float
+
+    static func parse() -> CLIArgs? {
+        var modelPath: URL?
+        var imagePath: URL?
+        var outputPath: URL?
+        var radius: Float = 5.0
+
+        var i = 1
+        while i < CommandLine.arguments.count {
+            let arg = CommandLine.arguments[i]
+            switch arg {
+            case "-m", "--model":
+                i += 1; guard i < CommandLine.arguments.count else { return nil }
+                modelPath = URL(fileURLWithPath: CommandLine.arguments[i])
+            case "-i", "--input":
+                i += 1; guard i < CommandLine.arguments.count else { return nil }
+                imagePath = URL(fileURLWithPath: CommandLine.arguments[i])
+            case "-o", "--output":
+                i += 1; guard i < CommandLine.arguments.count else { return nil }
+                outputPath = URL(fileURLWithPath: CommandLine.arguments[i])
+            case "-r", "--radius":
+                i += 1; guard i < CommandLine.arguments.count else { return nil }
+                radius = Float(CommandLine.arguments[i]) ?? 5.0
+            case "-h", "--help":
+                printUsage(); return nil
+            default: break
+            }
+            i += 1
+        }
+
+        guard let m = modelPath, let img = imagePath, let out = outputPath else {
+            printUsage(); return nil
+        }
+        return CLIArgs(modelPath: m, imagePath: img, outputPath: out, radius: radius)
+    }
+
+    static func printUsage() {
+        let name = CommandLine.arguments[0].components(separatedBy: "/").last ?? "panorama_splat"
+        print("""
+        Usage: \(name) -m <model> -i <image> -o <output.ply> [-r radius]
+
+        Convert equirectangular panoramas to 3D Gaussian splat PLY files.
+
+        Options:
+          -m, --model PATH     Path to DAP CoreML model (.mlpackage)
+          -i, --input PATH     Path to equirectangular panorama (2:1 ratio)
+          -o, --output PATH    Output PLY file path
+          -r, --radius FLOAT   Sphere radius in world units (default: 5.0)
+          -h, --help           Show this help
+        """)
+    }
+}
+
+// MARK: - CoreML Depth Inference
+
+func compileModelIfNeeded(at url: URL) throws -> URL {
+    let ext = url.pathExtension.lowercased()
+    guard ext == "mlpackage" || ext == "mlmodel" || ext == "mlmodelc" else {
+        fatalError("Unsupported model format: \(ext)")
+    }
+    guard ext != "mlmodelc" else { return url }
+
+    let cacheDir = FileManager.default.temporaryDirectory
+        .appendingPathComponent("PanoramaSplatCache")
+    try FileManager.default.createDirectory(at: cacheDir, withIntermediateDirectories: true)
+
+    let compiled = cacheDir.appendingPathComponent("\(url.deletingPathExtension().lastPathComponent).mlmodelc")
+
+    if FileManager.default.fileExists(atPath: compiled.path) {
+        if let src = try? FileManager.default.attributesOfItem(atPath: url.path)[.modificationDate] as? Date,
+           let cch = try? FileManager.default.attributesOfItem(atPath: compiled.path)[.modificationDate] as? Date,
+           cch >= src {
+            return compiled
+        }
+        try? FileManager.default.removeItem(at: compiled)
+    }
+
+    print("  Compiling CoreML model ...")
+    let t = CFAbsoluteTimeGetCurrent()
+    let tmp = try MLModel.compileModel(at: url)
+    try? FileManager.default.removeItem(at: compiled)
+    try FileManager.default.moveItem(at: tmp, to: compiled)
+    print("  Compiled in \(String(format: "%.1fs", CFAbsoluteTimeGetCurrent() - t))")
+    return compiled
+}
+
+func runDepthInference(modelURL: URL, image: CGImage) throws -> (depths: [Float32], width: Int, height: Int) {
+    let compiled = try compileModelIfNeeded(at: modelURL)
+    let config = MLModelConfiguration()
+    config.computeUnits = .all
+    let model = try MLModel(contentsOf: compiled, configuration: config)
+    let vnModel = try VNCoreMLModel(for: model)
+
+    let request = VNCoreMLRequest(model: vnModel) { _, error in
+        if let error { fatalError("Inference error: \(error)") }
+    }
+    request.imageCropAndScaleOption = .scaleFit
+
+    let handler = VNImageRequestHandler(cgImage: image, options: [:])
+    try handler.perform([request])
+
+    guard let observations = request.results as? [VNCoreMLFeatureValueObservation],
+          let ma = observations.first?.featureValue.multiArrayValue else {
+        fatalError("No depth output from model")
+    }
+
+    let h = ma.shape[2].intValue
+    let w = ma.shape[3].intValue
+    let planeStride = ma.strides[2].intValue
+    let ptr = ma.dataPointer.bindMemory(to: Float32.self, capacity: h * w)
+
+    var depths = [Float32](repeating: 0, count: h * w)
+    for row in 0..<h {
+        let src = ptr.advanced(by: row * planeStride)
+        let dst = depths.withUnsafeMutableBufferPointer { $0.baseAddress!.advanced(by: row * w) }
+        memcpy(dst, src, w * MemoryLayout<Float32>.stride)
+    }
+
+    return (depths, w, h)
+}
+
+// MARK: - Image Pixel Loading
+
+/// Load image as RGBA pixels resized to target dimensions.
+func loadImagePixels(_ image: CGImage, targetW: Int, targetH: Int) -> [UInt8] {
+    let ci = CIImage(cgImage: image)
+    let ctx = CIContext()
+
+    let scaled = ci.transformed(by: CGAffineTransform(scaleX: CGFloat(targetW) / ci.extent.width,
+                                                      y:  CGFloat(targetH) / ci.extent.height))
+    guard let resized = ctx.createCGImage(scaled, from: CGRect(x: 0, y: 0, width: targetW, height: targetH)) else {
+        fatalError("Failed to resize image to \(targetW)x\(targetH)")
+    }
+
+    let bpp = 4
+    let bpr = bpp * targetW
+    var pixels = [UInt8](repeating: 0, count: targetH * bpr)
+    let cs = CGColorSpaceCreateDeviceRGB()
+
+    guard let gctx = CGContext(data: &pixels, width: targetW, height: targetH,
+                               bitsPerComponent: 8, bytesPerRow: bpr, space: cs,
+                               bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else {
+        fatalError("Failed to create bitmap context")
+    }
+    gctx.draw(resized, in: CGRect(x: 0, y: 0, width: targetW, height: targetH))
+    return pixels
+}
+
+// MARK: - Equirectangular to 3D Projection
+
+func equiToSphereDirection(u: Float, v: Float, width: Int, height: Int) -> (x: Float, y: Float, z: Float) {
+    let lon = (u / Float(width) - 0.5) * 2.0 * Float.pi
+    let lat = (0.5 - v / Float(height)) * Float.pi
+    let cosLat = cos(lat)
+    return (cosLat * cos(lon), sin(lat), cosLat * sin(lon))
+}
+
+// MARK: - PLY Export (binary_little_endian, matches Sharp format)
+
+func writePLY(gaussians: [(x: Float, y: Float, z: Float,
+                            f0: Float, f1: Float, f2: Float,
+                            opacity: Float,
+                            s0: Float, s1: Float, s2: Float,
+                            q0: Float, q1: Float, q2: Float, q3: Float)],
+              focalLength: Float, imageW: Int, imageH: Int,
+              to url: URL) throws {
+
+    var data = Data()
+
+    func a(_ str: String) {
+        data.append(str.data(using: .ascii)!)
+    }
+
+    func f(_ v: Float) {
+        var vv = v; data.append(Data(bytes: &vv, count: 4))
+    }
+
+    func i32(_ v: Int32) {
+        var vv = v; data.append(Data(bytes: &vv, count: 4))
+    }
+
+    func u32(_ v: UInt32) {
+        var vv = v; data.append(Data(bytes: &vv, count: 4))
+    }
+
+    func u8(_ v: UInt8) {
+        var vv = v; data.append(Data(bytes: &vv, count: 1))
+    }
+
+    let n = gaussians.count
+
+    // --- Header ---
+    a("ply\n")
+    a("format binary_little_endian 1.0\n")
+    a("element vertex \(n)\n")
+    a("property float x\nproperty float y\nproperty float z\n")
+    a("property float f_dc_0\nproperty float f_dc_1\nproperty float f_dc_2\n")
+    a("property float opacity\n")
+    a("property float scale_0\nproperty float scale_1\nproperty float scale_2\n")
+    a("property float rot_0\nproperty float rot_1\nproperty float rot_2\nproperty float rot_3\n")
+    a("element extrinsic 16\nproperty float extrinsic\n")
+    a("element intrinsic 9\nproperty float intrinsic\n")
+    a("element image_size 2\nproperty uint image_size\n")
+    a("element frame 2\nproperty int frame\n")
+    a("element disparity 2\nproperty float disparity\n")
+    a("element color_space 1\nproperty uchar color_space\n")
+    a("element version 3\nproperty uchar version\n")
+    a("end_header\n")
+
+    // --- Vertex data ---
+    var disparities: [Float] = []
+    for g in gaussians {
+        f(g.x); f(g.y); f(g.z)
+        f(g.f0); f(g.f1); f(g.f2)
+        f(g.opacity)
+        f(g.s0); f(g.s1); f(g.s2)
+        f(g.q0); f(g.q1); f(g.q2); f(g.q3)
+        if g.z > 1e-6 { disparities.append(1.0 / g.z) }
+    }
+
+    // --- Extrinsic (identity 4x4) ---
+    let id: [Float] = [1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1]
+    for v in id { f(v) }
+
+    // --- Intrinsic (3x3) ---
+    f(focalLength); f(0);      f(Float(imageW) * 0.5)
+    f(0);         f(focalLength); f(Float(imageH) * 0.5)
+    f(0);         f(0);              f(1)
+
+    // --- Image size ---
+    u32(UInt32(imageW)); u32(UInt32(imageH))
+
+    // --- Frame ---
+    i32(1); i32(Int32(n))
+
+    // --- Disparity quantiles ---
+    disparities.sort()
+    let d10 = disparities.isEmpty ? 0.0 : disparities[min(Int(Float(disparities.count) * 0.1), disparities.count - 1)]
+    let d90 = disparities.isEmpty ? 1.0 : disparities[min(Int(Float(disparities.count) * 0.9), disparities.count - 1)]
+    f(d10); f(d90)
+
+    // --- Color space (sRGB = 1) ---
+    u8(1)
+
+    // --- Version ---
+    u8(1); u8(5); u8(0)
+
+    try data.write(to: url, options: .atomic)
+}
+
+// MARK: - Depth Map Seam Fix
+
+/// Equirectangular panoramas wrap at the left/right edges, which creates a
+/// seam artifact in the depth prediction.  Fix by:
+///   1. Rolling the depth map horizontally by W/2 so the seam lands in the middle
+///   2. Blending a narrow band around the (now-centred) seam with values from
+///      the opposite edge (which were originally adjacent in the panorama)
+///   3. Rolling back to the original orientation
+func fixDepthSeam(_ depths: [Float32], width: Int, height: Int, band: Int = 32) -> [Float32] {
+    let half = width / 2
+
+    // Step 1: roll left by half so seam is at column `half`
+    var shifted = [Float32](repeating: 0, count: width * height)
+    for row in 0..<height {
+        for col in 0..<width {
+            shifted[row * width + col] = depths[row * width + ((col + half) % width)]
+        }
+    }
+
+    // Step 2: cosine-blend a band around the seam with the opposite edge
+    let seamLeft = half - band
+    let seamRight = half + band
+    for row in 0..<height {
+        for col in seamLeft..<seamRight {
+            let offset = col - seamLeft           // 0 .. 2*band-1
+            let t = Float(offset) / Float(2 * band)
+            let w = 0.5 + 0.5 * cos(t * Float.pi) // 1 → 0 → 1
+
+            let srcA = shifted[row * width + col]
+            let srcB = shifted[row * width + ((col + band) % width)]
+            shifted[row * width + col] = srcA * w + srcB * (1.0 - w)
+        }
+    }
+
+    // Step 3: roll back (right by half) and return
+    var result = [Float32](repeating: 0, count: width * height)
+    for row in 0..<height {
+        for col in 0..<width {
+            result[row * width + col] = shifted[row * width + ((col - half + width) % width)]
+        }
+    }
+    return result
+}
+
+// MARK: - Main Pipeline
+
+func main() {
+    guard let args = CLIArgs.parse() else { exit(1) }
+
+    print("Loading image ...")
+    guard let nsImg = NSImage(contentsOf: args.imagePath) else {
+        fatalError("Cannot load image: \(args.imagePath.path)")
+    }
+    guard let cgImg = nsImg.cgImage(forProposedRect: nil, context: nil, hints: nil) else {
+        fatalError("Cannot convert image to CGImage")
+    }
+    print("  Image: \(cgImg.width)x\(cgImg.height)")
+
+    print("Running depth inference ...")
+    let t0 = CFAbsoluteTimeGetCurrent()
+    let (rawDepths, dW, dH) = try! runDepthInference(modelURL: args.modelPath, image: cgImg)
+    let dt = CFAbsoluteTimeGetCurrent() - t0
+    print("  Depth: \(dW)x\(dH) in \(String(format: "%.2fs", dt))")
+
+    print("Fixing depth seam ...")
+    let depths = fixDepthSeam(rawDepths, width: dW, height: dH)
+
+    print("Loading image pixels ...")
+    let pixels = loadImagePixels(cgImg, targetW: dW, targetH: dH)
+
+    let radius = args.radius
+    let coeffSH0 = sqrt(1.0 / (4.0 * Float.pi))
+    // Base angular footprint of one pixel (used as scale factor per-splat)
+    let pixelFootprint = radius * Float.pi / Float(max(dW, dH))
+    let uniformOpacity = Float(log(0.85 / (1.0 - 0.85)))  // logit(0.85) ≈ 1.96
+
+    print("Generating \(dW * dH) Gaussians ...")
+    var gaussians: [(x: Float, y: Float, z: Float,
+                      f0: Float, f1: Float, f2: Float,
+                      opacity: Float,
+                      s0: Float, s1: Float, s2: Float,
+                      q0: Float, q1: Float, q2: Float, q3: Float)] = []
+    gaussians.reserveCapacity(dW * dH)
+
+    for v in 0..<dH {
+        for u in 0..<dW {
+            let idx = v * dW + u
+            let depth = depths[idx]
+
+            // Skip zero-depth pixels (invalid / background)
+            guard depth > 0.01 else { continue }
+
+            var dir = equiToSphereDirection(u: Float(u), v: Float(v), width: dW, height: dH)
+            // Flip 180° (panorama was upside down — invert Y axis)
+            dir.y = -dir.y
+
+            let r = depth * radius
+            let px = dir.x * r
+            let py = dir.y * r
+            let pz = dir.z * r
+
+            // Scale proportional to distance — far splats are larger, near splats are tighter
+            let splatScale = Float(log(pixelFootprint * depth))
+
+            // Color from image pixel (RGBA)
+            let pidx = idx * 4
+            let rr = Float(pixels[pidx]) / 255.0
+            let gg = Float(pixels[pidx + 1]) / 255.0
+            let bb = Float(pixels[pidx + 2]) / 255.0
+
+            // RGB -> SH0
+            let f0 = (rr - 0.5) / coeffSH0
+            let f1 = (gg - 0.5) / coeffSH0
+            let f2 = (bb - 0.5) / coeffSH0
+
+            gaussians.append((
+                x: px, y: py, z: pz,
+                f0: f0, f1: f1, f2: f2,
+                opacity: uniformOpacity,
+                s0: splatScale, s1: splatScale, s2: splatScale,
+                q0: 1.0, q1: 0.0, q2: 0.0, q3: 0.0
+            ))
+        }
+    }
+
+    print("  Valid Gaussians: \(gaussians.count) (filtered \(dW * dH - gaussians.count) zero-depth pixels)")
+
+    print("Saving PLY ...")
+    let focal = Float(dW)  // panoramic focal ≈ image width
+    try! writePLY(gaussians: gaussians, focalLength: focal, imageW: dW, imageH: dH, to: args.outputPath)
+
+    let size = (try? FileManager.default.attributesOfItem(atPath: args.outputPath.path)[.size] as? UInt)?.description ?? "?"
+    print("  Saved \(args.outputPath.path) (\(size) bytes)")
+    print("Done!")
+}
+
+main()

README.md

@@ -1,5 +1,5 @@
 ---
-license: mit
+license: apache-2.0
 tags:
   - coreml
   - depth-estimation
@@ -72,6 +72,33 @@ Options:
 
 The model is automatically compiled on first use and cached for subsequent runs.
 
+## Quick Start — 360° Gaussian Splats (macOS)
+
+![](test/360splat.gif)
+
+Convert an equirectangular panorama directly into a 3D Gaussian splat `.ply` file — one Gaussian per pixel, compatible with standard 3DGS viewers:
+
+```bash
+# Compile
+swiftc -O -o panorama_splat PanoramaSplat.swift \
+    -framework CoreML -framework Vision -framework CoreImage \
+    -framework CoreGraphics -framework AppKit
+
+# Generate a Gaussian splat PLY
+./panorama_splat -m DAPModel.mlpackage -i test/test.png -o scene.ply -r 5.0
+```
+
+Options:
+
+| Flag | Description |
+|---|---|
+| `-m, --model PATH` | Path to `DAPModel.mlpackage` |
+| `-i, --input PATH` | Input equirectangular panorama (2:1 aspect ratio) |
+| `-o, --output PATH` | Output PLY file |
+| `-r, --radius FLOAT` | Sphere radius in world units (default: `5.0`) |
+
+The PLY uses the same binary format as [SHARP](https://github.com/apple/ml-sharp), with per-pixel positions projected onto a sphere using estimated depth, image-derived colors (SH0), uniform scale/opacity, and identity quaternions.
+
 ## Quick Start — Xcode (iOS / macOS)
 
 Add `DAPModel.mlpackage` to your Xcode project (Xcode auto-generates the `DAPModel` Swift class), then use the included `DepthPredictor.swift`:

scene.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9d3bb0ec0e1944be20f57fa5f95111868c55dc0ff0407970f8249e4daeb8cd6
+size 117439574

test_output/pytorch_depth.npy

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa87a4736dec10629a3654144e7aab4b417d7ac0ce81481525915094763aa4b9
-size 2097280
+oid sha256:10c5b39aa3f1220448efd33e9b3655c2f3c642a5a4fa1d2e25f76e37dfffce93
+size 8388736