Building a Native macOS GUI for Neovim

Why Build a Native GUI?

Terminal emulators are great, but they come with limitations:

• Font rendering: Limited control over ligatures, line spacing, and font variants
• Keyboard handling: Modifier keys often conflict with terminal escape sequences
• System integration: No native clipboard, drag-and-drop, or service menu support
• Performance: Extra layer between your editor and the display

A native GUI solves these problems while preserving Neovim's extensibility.

Requirements

System Requirements

• macOS 14.0+ (Sonoma) for modern SwiftUI features
• Swift 5.9+ for actor concurrency
• Neovim 0.9+ with --embed support

Dependencies

• MessagePack.swift (v4.0+) for RPC serialization

Architecture Overview

The architecture follows a layered approach with clear separation of concerns:

Component Communication Flow

Core Implementation

1. Process Management

The first challenge is launching Neovim as a subprocess. The --embed flag tells Neovim to communicate via stdin/stdout instead of rendering to a terminal:

public final class NvimProcess {
    private var process: Process?
    private var stdinPipe: Pipe?
    private var stdoutPipe: Pipe?

    public func launch(
        nvimPath: String? = nil,
        args: [String] = [],
        cwd: URL? = nil
    ) throws {
        let resolvedPath = nvimPath ?? Self.findNvim()
        guard let nvimExecutable = resolvedPath else {
            throw NvimProcessError.launchFailed("Could not find nvim")
        }

        let process = Process()
        process.executableURL = URL(fileURLWithPath: nvimExecutable)
        process.arguments = ["--embed"] + args

        // Inherit user's shell environment (PATH, etc.)
        process.environment = Self.userShellEnvironment()

        // Set up pipes for bidirectional communication
        let stdinPipe = Pipe()
        let stdoutPipe = Pipe()
        process.standardInput = stdinPipe
        process.standardOutput = stdoutPipe

        self.stdinPipe = stdinPipe
        self.stdoutPipe = stdoutPipe
        self.process = process

        try process.run()
    }
}

A critical detail: inherit the user's shell environment. Without this, Neovim won't find plugins, LSPs, or tools installed via Homebrew:

private static func userShellEnvironment() -> [String: String] {
    let shell = ProcessInfo.processInfo.environment["SHELL"] ?? "/bin/zsh"

    let process = Process()
    process.executableURL = URL(fileURLWithPath: shell)
    process.arguments = ["-l", "-c", "env"]  // Login shell to source profile

    let pipe = Pipe()
    process.standardOutput = pipe

    var environment = ProcessInfo.processInfo.environment

    try? process.run()
    process.waitUntilExit()

    // Parse env output into dictionary
    let data = pipe.fileHandleForReading.readDataToEndOfFile()
    if let output = String(data: data, encoding: .utf8) {
        for line in output.split(separator: "\n") {
            let parts = line.split(separator: "=", maxSplits: 1)
            if parts.count == 2 {
                environment[String(parts[0])] = String(parts[1])
            }
        }
    }

    return environment
}

2. MessagePack-RPC Protocol

Neovim uses MessagePack-RPC for communication. There are three message types:

The RPC layer handles serialization, message routing, and async request/response matching:

public final class MessagePackRPC {
    private let process: NvimProcess
    private var nextMsgID: UInt32 = 0
    private var pendingRequests: [UInt32: CheckedContinuation<MessagePackValue, Error>] = [:]
    private let lock = NSLock()

    /// Send a request and wait for response
    public func request(
        method: String,
        params: [MessagePackValue] = []
    ) async throws -> MessagePackValue {
        let msgid: UInt32 = lock.withLock {
            let id = nextMsgID
            nextMsgID += 1
            return id
        }

        let message: MessagePackValue = [
            .int(0),  // Request type
            .uint(UInt64(msgid)),
            .string(method),
            .array(params)
        ]

        let data = pack(message)
        try process.send(data)

        return try await withCheckedThrowingContinuation { continuation in
            lock.withLock {
                pendingRequests[msgid] = continuation
            }
        }
    }

    /// Send a notification (fire-and-forget)
    public func notify(method: String, params: [MessagePackValue] = []) throws {
        let message: MessagePackValue = [
            .int(2),  // Notification type
            .string(method),
            .array(params)
        ]
        try process.send(pack(message))
    }
}

3. UI Attachment

To receive rendering updates, attach as a UI client:

public func uiAttach(width: Int, height: Int) async throws {
    let opts: [MessagePackValue: MessagePackValue] = [
        .string("ext_linegrid"): .bool(true),  // Modern grid protocol
        .string("rgb"): .bool(true),           // 24-bit color
    ]

    _ = try await rpc.request(
        method: "nvim_ui_attach",
        params: [
            .int(Int64(width)),
            .int(Int64(height)),
            .map(opts)
        ]
    )
}

4. Grid-Based Rendering

Neovim sends UI updates as redraw notifications containing batched events. The grid is a 2D array of cells:

public struct Cell {
    public var text: String = " "
    public var highlightID: Int = 0
    public var isContinuation: Bool = false  // For double-width characters
}

public final class Grid {
    public private(set) var rows: Int
    public private(set) var columns: Int
    public private(set) var cells: [[Cell]]
    public var cursorRow: Int = 0
    public var cursorColumn: Int = 0

    public func updateRow(_ row: Int, startColumn: Int, cells newCells: [Cell]) {
        for (offset, cell) in newCells.enumerated() {
            let col = startColumn + offset
            guard col < columns else { break }
            cells[row][col] = cell
        }
    }

    public func scroll(top: Int, bottom: Int, left: Int, right: Int,
                       rows rowDelta: Int, columns colDelta: Int) {
        // Move content within the scroll region
        if rowDelta > 0 {
            for row in top..<(bottom - rowDelta) {
                for col in left..<right {
                    cells[row][col] = cells[row + rowDelta][col]
                }
            }
        }
        // Clear newly exposed rows...
    }
}

5. CoreText Rendering

For crisp text rendering with ligature support, use CoreText directly:

public final class Renderer {
    public var font: NSFont
    public private(set) var cellWidth: CGFloat = 0
    public private(set) var cellHeight: CGFloat = 0

    private var lineCache: [String: CTLine] = [:]

    public func draw(in context: CGContext, dirtyRect: CGRect) {
        // Fill background
        context.setFillColor(highlights.defaultBackground.cgColor)
        context.fill(dirtyRect)

        // Calculate visible rows
        let startRow = max(0, Int(floor(dirtyRect.minY / cellHeight)))
        let endRow = min(grid.rows, Int(ceil(dirtyRect.maxY / cellHeight)))

        for row in startRow..<endRow {
            drawRow(row, in: context)
        }
    }

    private func drawRow(_ row: Int, in context: CGContext) {
        // Group consecutive cells with same highlight for batched drawing
        var text = ""
        for col in 0..<grid.columns {
            if let cell = grid[row, col] {
                text += cell.text.isEmpty ? " " : cell.text
            }
        }

        // Create attributed string with font and colors
        let attributes: [NSAttributedString.Key: Any] = [
            .font: font,
            .foregroundColor: highlight.effectiveForeground,
            .ligature: useLigatures ? 1 : 0
        ]

        let line = CTLineCreateWithAttributedString(
            NSAttributedString(string: text, attributes: attributes)
        )

        // Draw at correct position (flipped coordinates)
        context.saveGState()
        context.translateBy(x: 0, y: y + ascent)
        context.scaleBy(x: 1, y: -1)
        CTLineDraw(line, context)
        context.restoreGState()
    }
}

6. Keyboard Input Translation

macOS keyboard events need translation to Vim notation:

public struct InputHandler {
    public static func convertKeyEvent(_ event: NSEvent) -> String? {
        var keys = ""
        let mods = event.modifierFlags

        // Build modifier prefix
        if mods.contains(.control) { keys += "C-" }
        if mods.contains(.option)  { keys += "M-" }  // Meta/Alt
        if mods.contains(.shift)   { /* handled by character case */ }

        // Map special keys
        switch event.keyCode {
        case 36:  return wrapMods(keys, "CR")     // Return
        case 51:  return wrapMods(keys, "BS")     // Backspace
        case 53:  return wrapMods(keys, "Esc")    // Escape
        case 48:  return wrapMods(keys, "Tab")    // Tab
        case 123: return wrapMods(keys, "Left")   // Arrow keys
        case 124: return wrapMods(keys, "Right")
        case 125: return wrapMods(keys, "Down")
        case 126: return wrapMods(keys, "Up")
        default: break
        }

        // Regular characters
        if let chars = event.characters, !chars.isEmpty {
            if keys.isEmpty {
                return chars  // No modifiers, send raw
            }
            return "<\(keys)\(chars)>"
        }

        return nil
    }

    private static func wrapMods(_ mods: String, _ key: String) -> String {
        if mods.isEmpty { return "<\(key)>" }
        return "<\(mods)\(key)>"
    }
}

Key Challenges & Solutions

Challenge 1: Double-Width Characters (CJK)

Chinese, Japanese, and Korean characters occupy two cells. The grid must track "continuation" cells:

// When parsing grid_line events:
if character.unicodeScalars.first?.properties.isEastAsianWide == true {
    cells.append(Cell(text: character, highlightID: hlID))
    cells.append(Cell(text: "", highlightID: hlID, isContinuation: true))
}

Challenge 2: Async/Await with MessagePack-RPC

Swift's concurrency model maps well to RPC. Use CheckedContinuation to bridge callback-based I/O:

return try await withCheckedThrowingContinuation { continuation in
    lock.withLock {
        pendingRequests[msgid] = continuation
    }
}

// Later, when response arrives:
if let continuation = pendingRequests.removeValue(forKey: msgid) {
    continuation.resume(returning: result)
}

Challenge 3: Scroll Performance

Neovim sends grid_scroll events rather than redrawing the entire screen. Implement efficient scrolling:

Challenge 4: Mode-Aware Paste

Pasting text differs by mode. Use nvim_paste for proper bracketed paste:

private func handlePaste() async {
    guard let clipboard = NSPasteboard.general.string(forType: .string) else { return }

    let mode = try? await api.callFunction("mode", args: [])

    if mode == "t" {  // Terminal mode
        _ = try await api.input(clipboard)  // Raw input
    } else {
        // Bracketed paste handles indentation correctly
        _ = try await api.paste(clipboard, crlf: false, phase: -1)
    }
}

Challenge 5: Font Resolution

Neovim's guifont format (FontName:h14) needs parsing:

private func parseGuiFont(_ spec: String) -> NSFont? {
    let parts = spec.split(separator: ":")
    var fontName = String(parts.first ?? "")
        .replacingOccurrences(of: "_", with: " ")
    var fontSize: CGFloat = 14

    for part in parts.dropFirst() {
        if part.hasPrefix("h"), let size = Double(part.dropFirst()) {
            fontSize = CGFloat(size)
        }
    }

    // Try exact name, then common Nerd Font variants
    return NSFont(name: fontName, size: fontSize)
        ?? NSFont(name: fontName + " Nerd Font Mono", size: fontSize)
}

Project Structure

TrinityApp/
├── TrinityApp.swift        # Entry point, AppDelegate
├── ContentView.swift       # Main UI, AppState coordinator
├── Commands.swift          # Menu commands
└── SettingsView.swift      # Preferences

Packages/
├── NvimView/
│   ├── NvimView.swift      # Main NSView component
│   ├── Grid/
│   │   ├── Cell.swift
│   │   ├── Grid.swift
│   │   └── Highlight.swift
│   ├── RPC/
│   │   ├── NvimProcess.swift
│   │   ├── MessagePackRPC.swift
│   │   └── NvimAPI.swift
│   └── UI/
│       ├── Renderer.swift
│       ├── InputHandler.swift
│       └── UIEventHandler.swift
├── FileBrowser/            # File tree sidebar
├── MarkdownPreview/        # Live preview pane
└── Commons/                # Shared utilities

Performance Optimizations

1. CTLine Caching: Cache rendered text lines to avoid repeated CoreText calls
2. Dirty Rect Rendering: Only redraw affected rows
3. Scroll Throttling: Limit scroll events to ~60fps
4. Batched Updates: Process multiple grid updates before redrawing

// CTLine cache for performance
private var lineCache: [String: CTLine] = [:]
private let maxCacheSize = 2000

private func cachedLine(for text: String, attributes: [NSAttributedString.Key: Any]) -> CTLine {
    let key = "\(text):\(attributes.hashValue)"

    if let cached = lineCache[key] {
        return cached
    }

    if lineCache.count > maxCacheSize {
        lineCache.removeAll(keepingCapacity: true)
    }

    let line = CTLineCreateWithAttributedString(
        NSAttributedString(string: text, attributes: attributes)
    )
    lineCache[key] = line
    return line
}

Conclusion

Building a native Neovim GUI is a rewarding project that combines:

• Systems programming: Process management, pipes, binary protocols
• Graphics: CoreText rendering, coordinate systems, dirty rect optimization
• Concurrency: Async/await, continuation-based I/O
• macOS integration: NSView, keyboard handling, clipboard

The result is a fast, native editor that feels at home on macOS while preserving everything that makes Neovim powerful. Your init.lua, plugins, and muscle memory all work unchanged—but now with proper font rendering, smooth scrolling, and native keyboard shortcuts.

Want to see it in action? Check out Trinity for screenshots and to try the finished product.

Built with Swift, SwiftUI, and a deep appreciation for Neovim.